@midscene/core 1.17.5 → 1.17.7

package/dist/es/ai-model/service-caller/openai/siwc/refresh.mjs

@@ -1,12 +1,14 @@
-import { createRemoteJWKSet, jwtVerify } from "jose";
+import { jwtVerify } from "jose";
 import { tokenResponseError, withSiwcStage } from "./errors.mjs";
+import { createSiwcJWKSet, fetchSiwc } from "./network.mjs";
 async function refreshSiwcCredentials(saved, signal, dependencies) {
     const { client_id: clientId, refresh_token: refreshToken } = saved;
     if (!clientId || !refreshToken) throw new Error('Missing client ID or refresh token.');
+    const request = dependencies?.fetch ?? fetchSiwc;
     const issuer = 'https://auth.openai.com';
     signal.throwIfAborted();
     const response = await withSiwcStage('Token refresh', async ()=>{
-        const response = await (dependencies?.fetch ?? fetch)(`${issuer}/api/accounts/oauth/token`, {
+        const response = await request(`${issuer}/api/accounts/oauth/token`, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/x-www-form-urlencoded'
@@ -29,7 +31,7 @@ async function refreshSiwcCredentials(saved, signal, dependencies) {
     if (!scopes.includes('chatgpt.tokens.use.direct')) throw new Error('ChatGPT plan usage is no longer authorized.');
     if (void 0 !== tokens.id_token) {
         if ('string' != typeof tokens.id_token || !tokens.id_token) throw new Error('Invalid refreshed ID token.');
-        const keys = dependencies?.keys ?? createRemoteJWKSet(new URL(`${issuer}/.well-known/jwks.json`));
+        const keys = dependencies?.keys ?? createSiwcJWKSet(issuer, signal, request);
         const { payload } = await withSiwcStage('ID token verification', ()=>jwtVerify(tokens.id_token, keys, {
                 issuer,
                 audience: saved.client_id,

package/dist/es/ai-model/service-caller/openai/siwc/refresh.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/openai/siwc/refresh.mjs","sources":["../../../../../../src/ai-model/service-caller/openai/siwc/refresh.ts"],"sourcesContent":["import { type JWTVerifyGetKey, createRemoteJWKSet, jwtVerify } from 'jose';\nimport { tokenResponseError, withSiwcStage } from './errors';\ninterface RefreshableCredentials {\n  client_id?: string;\n  refresh_token?: string;\n  subject?: string;\n  id_token?: string;\n}\n\nexport async function refreshSiwcCredentials<T extends RefreshableCredentials>(\n  saved: T,\n  signal: AbortSignal,\n  dependencies?: { fetch: typeof fetch; keys: JWTVerifyGetKey },\n) {\n  const { client_id: clientId, refresh_token: refreshToken } = saved;\n  if (!clientId || !refreshToken) {\n    throw new Error('Missing client ID or refresh token.');\n  }\n  const issuer = 'https://auth.openai.com';\n  signal.throwIfAborted();\n  const response = await withSiwcStage('Token refresh', async () => {\n    const response = await (dependencies?.fetch ?? fetch)(\n      `${issuer}/api/accounts/oauth/token`,\n      {\n        method: 'POST',\n        headers: { 'Content-Type': 'application/x-www-form-urlencoded' },\n        body: new URLSearchParams({\n          grant_type: 'refresh_token',\n          client_id: clientId,\n          refresh_token: refreshToken,\n          resource: 'https://api.openai.com/v1',\n        }),\n        signal,\n        redirect: 'error',\n      },\n    );\n    if (!response.ok) {\n      throw await tokenResponseError(response);\n    }\n    return response;\n  });\n  const tokens = await withSiwcStage('Refresh response parsing', () =>\n    response.json(),\n  );\n  if (\n    !tokens ||\n    typeof tokens.access_token !== 'string' ||\n    !tokens.access_token.trim() ||\n    typeof tokens.refresh_token !== 'string' ||\n    !tokens.refresh_token.trim() ||\n    typeof tokens.expires_in !== 'number' ||\n    !Number.isFinite(tokens.expires_in) ||\n    tokens.expires_in <= 0 ||\n    typeof tokens.token_type !== 'string' ||\n    tokens.token_type.toLowerCase() !== 'bearer' ||\n    typeof tokens.scope !== 'string'\n  ) {\n    throw new Error('OpenAI returned an invalid refresh response.');\n  }\n  const scopes = tokens.scope.split(/\\s+/);\n  if (!scopes.includes('chatgpt.tokens.use.direct')) {\n    throw new Error('ChatGPT plan usage is no longer authorized.');\n  }\n  if (tokens.id_token !== undefined) {\n    if (typeof tokens.id_token !== 'string' || !tokens.id_token) {\n      throw new Error('Invalid refreshed ID token.');\n    }\n    const keys =\n      dependencies?.keys ??\n      createRemoteJWKSet(new URL(`${issuer}/.well-known/jwks.json`));\n    const { payload } = await withSiwcStage('ID token verification', () =>\n      jwtVerify(tokens.id_token, keys, {\n        issuer,\n        audience: saved.client_id,\n        algorithms: ['RS256'],\n        // Some CI machines have incorrect system clocks. Do not compare ID token\n        // exp/nbf claims against local time when accepting newly returned tokens.\n        // Keep signature, issuer, audience and required-claim validation.\n        clockTolerance: Number.POSITIVE_INFINITY,\n        requiredClaims: ['sub', 'exp', 'iat'],\n      }),\n    );\n    if (payload.sub !== saved.subject) {\n      throw new Error('Refreshed identity does not match the saved account.');\n    }\n  }\n  signal.throwIfAborted();\n  return {\n    ...saved,\n    access_token: tokens.access_token,\n    refresh_token: tokens.refresh_token,\n    id_token: tokens.id_token ?? saved.id_token,\n    scopes,\n    expires_at: Date.now() + tokens.expires_in * 1000,\n  };\n}\n"],"names":["refreshSiwcCredentials","saved","signal","dependencies","clientId","refreshToken","Error","issuer","response","withSiwcStage","fetch","URLSearchParams","tokenResponseError","tokens","Number","scopes","undefined","keys","createRemoteJWKSet","URL","payload","jwtVerify","Date"],"mappings":";;AASO,eAAeA,uBACpBC,KAAQ,EACRC,MAAmB,EACnBC,YAA6D;IAE7D,MAAM,EAAE,WAAWC,QAAQ,EAAE,eAAeC,YAAY,EAAE,GAAGJ;IAC7D,IAAI,CAACG,YAAY,CAACC,cAChB,MAAM,IAAIC,MAAM;IAElB,MAAMC,SAAS;IACfL,OAAO,cAAc;IACrB,MAAMM,WAAW,MAAMC,cAAc,iBAAiB;QACpD,MAAMD,WAAW,MAAOL,AAAAA,CAAAA,cAAc,SAASO,KAAI,EACjD,GAAGH,OAAO,yBAAyB,CAAC,EACpC;YACE,QAAQ;YACR,SAAS;gBAAE,gBAAgB;YAAoC;YAC/D,MAAM,IAAII,gBAAgB;gBACxB,YAAY;gBACZ,WAAWP;gBACX,eAAeC;gBACf,UAAU;YACZ;YACAH;YACA,UAAU;QACZ;QAEF,IAAI,CAACM,SAAS,EAAE,EACd,MAAM,MAAMI,mBAAmBJ;QAEjC,OAAOA;IACT;IACA,MAAMK,SAAS,MAAMJ,cAAc,4BAA4B,IAC7DD,SAAS,IAAI;IAEf,IACE,CAACK,UACD,AAA+B,YAA/B,OAAOA,OAAO,YAAY,IAC1B,CAACA,OAAO,YAAY,CAAC,IAAI,MACzB,AAAgC,YAAhC,OAAOA,OAAO,aAAa,IAC3B,CAACA,OAAO,aAAa,CAAC,IAAI,MAC1B,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxB,CAACC,OAAO,QAAQ,CAACD,OAAO,UAAU,KAClCA,OAAO,UAAU,IAAI,KACrB,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxBA,AAAoC,aAApCA,OAAO,UAAU,CAAC,WAAW,MAC7B,AAAwB,YAAxB,OAAOA,OAAO,KAAK,EAEnB,MAAM,IAAIP,MAAM;IAElB,MAAMS,SAASF,OAAO,KAAK,CAAC,KAAK,CAAC;IAClC,IAAI,CAACE,OAAO,QAAQ,CAAC,8BACnB,MAAM,IAAIT,MAAM;IAElB,IAAIO,AAAoBG,WAApBH,OAAO,QAAQ,EAAgB;QACjC,IAAI,AAA2B,YAA3B,OAAOA,OAAO,QAAQ,IAAiB,CAACA,OAAO,QAAQ,EACzD,MAAM,IAAIP,MAAM;QAElB,MAAMW,OACJd,cAAc,QACde,mBAAmB,IAAIC,IAAI,GAAGZ,OAAO,sBAAsB,CAAC;QAC9D,MAAM,EAAEa,OAAO,EAAE,GAAG,MAAMX,cAAc,yBAAyB,IAC/DY,UAAUR,OAAO,QAAQ,EAAEI,MAAM;gBAC/BV;gBACA,UAAUN,MAAM,SAAS;gBACzB,YAAY;oBAAC;iBAAQ;gBAIrB,gBAAgBa;gBAChB,gBAAgB;oBAAC;oBAAO;oBAAO;iBAAM;YACvC;QAEF,IAAIM,QAAQ,GAAG,KAAKnB,MAAM,OAAO,EAC/B,MAAM,IAAIK,MAAM;IAEpB;IACAJ,OAAO,cAAc;IACrB,OAAO;QACL,GAAGD,KAAK;QACR,cAAcY,OAAO,YAAY;QACjC,eAAeA,OAAO,aAAa;QACnC,UAAUA,OAAO,QAAQ,IAAIZ,MAAM,QAAQ;QAC3Cc;QACA,YAAYO,KAAK,GAAG,KAAKT,AAAoB,OAApBA,OAAO,UAAU;IAC5C;AACF"}
+{"version":3,"file":"ai-model/service-caller/openai/siwc/refresh.mjs","sources":["../../../../../../src/ai-model/service-caller/openai/siwc/refresh.ts"],"sourcesContent":["import { type JWTVerifyGetKey, jwtVerify } from 'jose';\nimport { tokenResponseError, withSiwcStage } from './errors';\nimport { createSiwcJWKSet, fetchSiwc } from './network';\ninterface RefreshableCredentials {\n  client_id?: string;\n  refresh_token?: string;\n  subject?: string;\n  id_token?: string;\n}\n\nexport async function refreshSiwcCredentials<T extends RefreshableCredentials>(\n  saved: T,\n  signal: AbortSignal,\n  dependencies?: { fetch: typeof fetch; keys?: JWTVerifyGetKey },\n) {\n  const { client_id: clientId, refresh_token: refreshToken } = saved;\n  if (!clientId || !refreshToken) {\n    throw new Error('Missing client ID or refresh token.');\n  }\n  const request = dependencies?.fetch ?? fetchSiwc;\n  const issuer = 'https://auth.openai.com';\n  signal.throwIfAborted();\n  const response = await withSiwcStage('Token refresh', async () => {\n    const response = await request(`${issuer}/api/accounts/oauth/token`, {\n      method: 'POST',\n      headers: { 'Content-Type': 'application/x-www-form-urlencoded' },\n      body: new URLSearchParams({\n        grant_type: 'refresh_token',\n        client_id: clientId,\n        refresh_token: refreshToken,\n        resource: 'https://api.openai.com/v1',\n      }),\n      signal,\n      redirect: 'error',\n    });\n    if (!response.ok) {\n      throw await tokenResponseError(response);\n    }\n    return response;\n  });\n  const tokens = await withSiwcStage('Refresh response parsing', () =>\n    response.json(),\n  );\n  if (\n    !tokens ||\n    typeof tokens.access_token !== 'string' ||\n    !tokens.access_token.trim() ||\n    typeof tokens.refresh_token !== 'string' ||\n    !tokens.refresh_token.trim() ||\n    typeof tokens.expires_in !== 'number' ||\n    !Number.isFinite(tokens.expires_in) ||\n    tokens.expires_in <= 0 ||\n    typeof tokens.token_type !== 'string' ||\n    tokens.token_type.toLowerCase() !== 'bearer' ||\n    typeof tokens.scope !== 'string'\n  ) {\n    throw new Error('OpenAI returned an invalid refresh response.');\n  }\n  const scopes = tokens.scope.split(/\\s+/);\n  if (!scopes.includes('chatgpt.tokens.use.direct')) {\n    throw new Error('ChatGPT plan usage is no longer authorized.');\n  }\n  if (tokens.id_token !== undefined) {\n    if (typeof tokens.id_token !== 'string' || !tokens.id_token) {\n      throw new Error('Invalid refreshed ID token.');\n    }\n    const keys =\n      dependencies?.keys ?? createSiwcJWKSet(issuer, signal, request);\n    const { payload } = await withSiwcStage('ID token verification', () =>\n      jwtVerify(tokens.id_token, keys, {\n        issuer,\n        audience: saved.client_id,\n        algorithms: ['RS256'],\n        // Some CI machines have incorrect system clocks. Do not compare ID token\n        // exp/nbf claims against local time when accepting newly returned tokens.\n        // Keep signature, issuer, audience and required-claim validation.\n        clockTolerance: Number.POSITIVE_INFINITY,\n        requiredClaims: ['sub', 'exp', 'iat'],\n      }),\n    );\n    if (payload.sub !== saved.subject) {\n      throw new Error('Refreshed identity does not match the saved account.');\n    }\n  }\n  signal.throwIfAborted();\n  return {\n    ...saved,\n    access_token: tokens.access_token,\n    refresh_token: tokens.refresh_token,\n    id_token: tokens.id_token ?? saved.id_token,\n    scopes,\n    expires_at: Date.now() + tokens.expires_in * 1000,\n  };\n}\n"],"names":["refreshSiwcCredentials","saved","signal","dependencies","clientId","refreshToken","Error","request","fetchSiwc","issuer","response","withSiwcStage","URLSearchParams","tokenResponseError","tokens","Number","scopes","undefined","keys","createSiwcJWKSet","payload","jwtVerify","Date"],"mappings":";;;AAUO,eAAeA,uBACpBC,KAAQ,EACRC,MAAmB,EACnBC,YAA8D;IAE9D,MAAM,EAAE,WAAWC,QAAQ,EAAE,eAAeC,YAAY,EAAE,GAAGJ;IAC7D,IAAI,CAACG,YAAY,CAACC,cAChB,MAAM,IAAIC,MAAM;IAElB,MAAMC,UAAUJ,cAAc,SAASK;IACvC,MAAMC,SAAS;IACfP,OAAO,cAAc;IACrB,MAAMQ,WAAW,MAAMC,cAAc,iBAAiB;QACpD,MAAMD,WAAW,MAAMH,QAAQ,GAAGE,OAAO,yBAAyB,CAAC,EAAE;YACnE,QAAQ;YACR,SAAS;gBAAE,gBAAgB;YAAoC;YAC/D,MAAM,IAAIG,gBAAgB;gBACxB,YAAY;gBACZ,WAAWR;gBACX,eAAeC;gBACf,UAAU;YACZ;YACAH;YACA,UAAU;QACZ;QACA,IAAI,CAACQ,SAAS,EAAE,EACd,MAAM,MAAMG,mBAAmBH;QAEjC,OAAOA;IACT;IACA,MAAMI,SAAS,MAAMH,cAAc,4BAA4B,IAC7DD,SAAS,IAAI;IAEf,IACE,CAACI,UACD,AAA+B,YAA/B,OAAOA,OAAO,YAAY,IAC1B,CAACA,OAAO,YAAY,CAAC,IAAI,MACzB,AAAgC,YAAhC,OAAOA,OAAO,aAAa,IAC3B,CAACA,OAAO,aAAa,CAAC,IAAI,MAC1B,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxB,CAACC,OAAO,QAAQ,CAACD,OAAO,UAAU,KAClCA,OAAO,UAAU,IAAI,KACrB,AAA6B,YAA7B,OAAOA,OAAO,UAAU,IACxBA,AAAoC,aAApCA,OAAO,UAAU,CAAC,WAAW,MAC7B,AAAwB,YAAxB,OAAOA,OAAO,KAAK,EAEnB,MAAM,IAAIR,MAAM;IAElB,MAAMU,SAASF,OAAO,KAAK,CAAC,KAAK,CAAC;IAClC,IAAI,CAACE,OAAO,QAAQ,CAAC,8BACnB,MAAM,IAAIV,MAAM;IAElB,IAAIQ,AAAoBG,WAApBH,OAAO,QAAQ,EAAgB;QACjC,IAAI,AAA2B,YAA3B,OAAOA,OAAO,QAAQ,IAAiB,CAACA,OAAO,QAAQ,EACzD,MAAM,IAAIR,MAAM;QAElB,MAAMY,OACJf,cAAc,QAAQgB,iBAAiBV,QAAQP,QAAQK;QACzD,MAAM,EAAEa,OAAO,EAAE,GAAG,MAAMT,cAAc,yBAAyB,IAC/DU,UAAUP,OAAO,QAAQ,EAAEI,MAAM;gBAC/BT;gBACA,UAAUR,MAAM,SAAS;gBACzB,YAAY;oBAAC;iBAAQ;gBAIrB,gBAAgBc;gBAChB,gBAAgB;oBAAC;oBAAO;oBAAO;iBAAM;YACvC;QAEF,IAAIK,QAAQ,GAAG,KAAKnB,MAAM,OAAO,EAC/B,MAAM,IAAIK,MAAM;IAEpB;IACAJ,OAAO,cAAc;IACrB,OAAO;QACL,GAAGD,KAAK;QACR,cAAca,OAAO,YAAY;QACjC,eAAeA,OAAO,aAAa;QACnC,UAAUA,OAAO,QAAQ,IAAIb,MAAM,QAAQ;QAC3Ce;QACA,YAAYM,KAAK,GAAG,KAAKR,AAAoB,OAApBA,OAAO,UAAU;IAC5C;AACF"}

package/dist/es/ai-model/service-caller/openai/siwc/token-manager.mjs

@@ -1,4 +1,5 @@
 import { getCredentialStorage } from "./credential-storage.mjs";
+import { fetchSiwc } from "./network.mjs";
 import { refreshSiwcCredentials } from "./refresh.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
@@ -16,19 +17,21 @@ class SiwcTokenManager {
         const record = await storage.read();
         return record.credentials.access_token;
     }
-    refreshAccessToken(previousAccessToken) {
-        this.pendingRefresh ??= this.refreshAndSave(previousAccessToken).finally(()=>{
+    refreshAccessToken(previousAccessToken, proxy = {}) {
+        this.pendingRefresh ??= this.refreshAndSave(previousAccessToken, proxy).finally(()=>{
             this.pendingRefresh = void 0;
         });
         return this.pendingRefresh;
     }
-    async refreshAndSave(previousAccessToken) {
+    async refreshAndSave(previousAccessToken, proxy) {
         const storage = await getCredentialStorage();
         const record = await storage.read();
         const saved = record.credentials;
         if (saved.access_token !== previousAccessToken) return saved.access_token;
         try {
-            const credentials = await refreshSiwcCredentials(saved, AbortSignal.timeout(30000));
+            const credentials = await refreshSiwcCredentials(saved, AbortSignal.timeout(30000), {
+                fetch: (input, init)=>fetchSiwc(input, init, proxy)
+            });
             await storage.save({
                 ...record,
                 updated_at: new Date().toISOString(),

package/dist/es/ai-model/service-caller/openai/siwc/token-manager.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/openai/siwc/token-manager.mjs","sources":["../../../../../../src/ai-model/service-caller/openai/siwc/token-manager.ts"],"sourcesContent":["import { getCredentialStorage } from './credential-storage';\nimport { refreshSiwcCredentials } from './refresh';\n\nclass SiwcTokenManager {\n  private pendingRefresh?: Promise<string>;\n\n  async getAccessToken(): Promise<string> {\n    const storage = await getCredentialStorage();\n    const record = await storage.read();\n    // Some CI machines have incorrect system clocks. Let the server reject\n    // expired tokens instead of comparing expires_at against local time.\n    return record.credentials.access_token;\n  }\n\n  /** Only refresh operations are shared; ordinary reads remain independent. */\n  refreshAccessToken(previousAccessToken: string): Promise<string> {\n    this.pendingRefresh ??= this.refreshAndSave(previousAccessToken).finally(\n      () => {\n        this.pendingRefresh = undefined;\n      },\n    );\n    return this.pendingRefresh;\n  }\n\n  private async refreshAndSave(previousAccessToken: string): Promise<string> {\n    // Re-read after acquiring the refresh slot: another caller may have saved\n    // a new token since this request first read its credentials.\n    const storage = await getCredentialStorage();\n    const record = await storage.read();\n    const saved = record.credentials;\n    if (saved.access_token !== previousAccessToken) {\n      return saved.access_token;\n    }\n    try {\n      const credentials = await refreshSiwcCredentials(\n        saved,\n        AbortSignal.timeout(30_000),\n      );\n      await storage.save({\n        ...record,\n        updated_at: new Date().toISOString(),\n        credentials,\n      });\n      return credentials.access_token;\n    } catch (cause) {\n      throw new Error(\n        `Unable to refresh SIWC credentials: ${cause instanceof Error ? cause.message : String(cause)} ${storage.reauthorizationHint}`,\n        { cause },\n      );\n    }\n  }\n}\n\nexport const siwcTokenManager = new SiwcTokenManager();\n"],"names":["SiwcTokenManager","storage","getCredentialStorage","record","previousAccessToken","undefined","saved","credentials","refreshSiwcCredentials","AbortSignal","Date","cause","Error","String","siwcTokenManager"],"mappings":";;;;;;;;;;;;AAGA,MAAMA;IAGJ,MAAM,iBAAkC;QACtC,MAAMC,UAAU,MAAMC;QACtB,MAAMC,SAAS,MAAMF,QAAQ,IAAI;QAGjC,OAAOE,OAAO,WAAW,CAAC,YAAY;IACxC;IAGA,mBAAmBC,mBAA2B,EAAmB;QAC/D,IAAI,CAAC,cAAc,KAAK,IAAI,CAAC,cAAc,CAACA,qBAAqB,OAAO,CACtE;YACE,IAAI,CAAC,cAAc,GAAGC;QACxB;QAEF,OAAO,IAAI,CAAC,cAAc;IAC5B;IAEA,MAAc,eAAeD,mBAA2B,EAAmB;QAGzE,MAAMH,UAAU,MAAMC;QACtB,MAAMC,SAAS,MAAMF,QAAQ,IAAI;QACjC,MAAMK,QAAQH,OAAO,WAAW;QAChC,IAAIG,MAAM,YAAY,KAAKF,qBACzB,OAAOE,MAAM,YAAY;QAE3B,IAAI;YACF,MAAMC,cAAc,MAAMC,uBACxBF,OACAG,YAAY,OAAO,CAAC;YAEtB,MAAMR,QAAQ,IAAI,CAAC;gBACjB,GAAGE,MAAM;gBACT,YAAY,IAAIO,OAAO,WAAW;gBAClCH;YACF;YACA,OAAOA,YAAY,YAAY;QACjC,EAAE,OAAOI,OAAO;YACd,MAAM,IAAIC,MACR,CAAC,oCAAoC,EAAED,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF,OAAO,CAAC,EAAEV,QAAQ,mBAAmB,EAAE,EAC9H;gBAAEU;YAAM;QAEZ;IACF;;QA9CA,uBAAQ,kBAAR;;AA+CF;AAEO,MAAMG,mBAAmB,IAAId"}
+{"version":3,"file":"ai-model/service-caller/openai/siwc/token-manager.mjs","sources":["../../../../../../src/ai-model/service-caller/openai/siwc/token-manager.ts"],"sourcesContent":["import { getCredentialStorage } from './credential-storage';\nimport { type SiwcProxyConfig, fetchSiwc } from './network';\nimport { refreshSiwcCredentials } from './refresh';\n\nclass SiwcTokenManager {\n  private pendingRefresh?: Promise<string>;\n\n  async getAccessToken(): Promise<string> {\n    const storage = await getCredentialStorage();\n    const record = await storage.read();\n    // Some CI machines have incorrect system clocks. Let the server reject\n    // expired tokens instead of comparing expires_at against local time.\n    return record.credentials.access_token;\n  }\n\n  /** Only refresh operations are shared; ordinary reads remain independent. */\n  refreshAccessToken(\n    previousAccessToken: string,\n    proxy: SiwcProxyConfig = {},\n  ): Promise<string> {\n    this.pendingRefresh ??= this.refreshAndSave(\n      previousAccessToken,\n      proxy,\n    ).finally(() => {\n      this.pendingRefresh = undefined;\n    });\n    return this.pendingRefresh;\n  }\n\n  private async refreshAndSave(\n    previousAccessToken: string,\n    proxy: SiwcProxyConfig,\n  ): Promise<string> {\n    // Re-read after acquiring the refresh slot: another caller may have saved\n    // a new token since this request first read its credentials.\n    const storage = await getCredentialStorage();\n    const record = await storage.read();\n    const saved = record.credentials;\n    if (saved.access_token !== previousAccessToken) {\n      return saved.access_token;\n    }\n    try {\n      const credentials = await refreshSiwcCredentials(\n        saved,\n        AbortSignal.timeout(30_000),\n        { fetch: (input, init) => fetchSiwc(input, init, proxy) },\n      );\n      await storage.save({\n        ...record,\n        updated_at: new Date().toISOString(),\n        credentials,\n      });\n      return credentials.access_token;\n    } catch (cause) {\n      throw new Error(\n        `Unable to refresh SIWC credentials: ${cause instanceof Error ? cause.message : String(cause)} ${storage.reauthorizationHint}`,\n        { cause },\n      );\n    }\n  }\n}\n\nexport const siwcTokenManager = new SiwcTokenManager();\n"],"names":["SiwcTokenManager","storage","getCredentialStorage","record","previousAccessToken","proxy","undefined","saved","credentials","refreshSiwcCredentials","AbortSignal","input","init","fetchSiwc","Date","cause","Error","String","siwcTokenManager"],"mappings":";;;;;;;;;;;;;AAIA,MAAMA;IAGJ,MAAM,iBAAkC;QACtC,MAAMC,UAAU,MAAMC;QACtB,MAAMC,SAAS,MAAMF,QAAQ,IAAI;QAGjC,OAAOE,OAAO,WAAW,CAAC,YAAY;IACxC;IAGA,mBACEC,mBAA2B,EAC3BC,QAAyB,CAAC,CAAC,EACV;QACjB,IAAI,CAAC,cAAc,KAAK,IAAI,CAAC,cAAc,CACzCD,qBACAC,OACA,OAAO,CAAC;YACR,IAAI,CAAC,cAAc,GAAGC;QACxB;QACA,OAAO,IAAI,CAAC,cAAc;IAC5B;IAEA,MAAc,eACZF,mBAA2B,EAC3BC,KAAsB,EACL;QAGjB,MAAMJ,UAAU,MAAMC;QACtB,MAAMC,SAAS,MAAMF,QAAQ,IAAI;QACjC,MAAMM,QAAQJ,OAAO,WAAW;QAChC,IAAII,MAAM,YAAY,KAAKH,qBACzB,OAAOG,MAAM,YAAY;QAE3B,IAAI;YACF,MAAMC,cAAc,MAAMC,uBACxBF,OACAG,YAAY,OAAO,CAAC,QACpB;gBAAE,OAAO,CAACC,OAAOC,OAASC,UAAUF,OAAOC,MAAMP;YAAO;YAE1D,MAAMJ,QAAQ,IAAI,CAAC;gBACjB,GAAGE,MAAM;gBACT,YAAY,IAAIW,OAAO,WAAW;gBAClCN;YACF;YACA,OAAOA,YAAY,YAAY;QACjC,EAAE,OAAOO,OAAO;YACd,MAAM,IAAIC,MACR,CAAC,oCAAoC,EAAED,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF,OAAO,CAAC,EAAEd,QAAQ,mBAAmB,EAAE,EAC9H;gBAAEc;YAAM;QAEZ;IACF;;QAtDA,uBAAQ,kBAAR;;AAuDF;AAEO,MAAMG,mBAAmB,IAAIlB"}

package/dist/es/ai-model/shared/json.mjs

@@ -1,9 +1,13 @@
 import { jsonrepair } from "jsonrepair";
 function extractJSONFromCodeBlock(response) {
     try {
-        const jsonMatch = response.match(/^\s*(\{[\s\S]*\})\s*$/);
+        JSON.parse(response);
+        return response.trim();
+    } catch  {}
+    try {
+        const jsonMatch = response.match(/^\s*(\{[\s\S]*\}|\[[\s\S]*\])\s*$/);
         if (jsonMatch) return jsonMatch[1];
-        const codeBlockMatch = response.match(/```(?:json)?\s*(\{[\s\S]*?\})\s*```/);
+        const codeBlockMatch = response.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
         if (codeBlockMatch) return codeBlockMatch[1];
         const jsonLikeMatch = response.match(/\{[\s\S]*\}/);
         if (jsonLikeMatch) return jsonLikeMatch[0];

package/dist/es/ai-model/shared/json.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/json.mjs","sources":["../../../../src/ai-model/shared/json.ts"],"sourcesContent":["import { jsonrepair } from 'jsonrepair';\n\n/**\n * Extract the JSON portion from a model response.\n *\n * This mainly handles fenced JSON like ```json { ... } ``` by removing the\n * fence. If that does not produce a JSON object, it falls back to a greedy regex\n * that extracts from the first \"{\" to the last \"}\". If that still fails, the\n * original response is returned.\n *\n * Expected model responses are JSON objects, possibly wrapped in markdown\n * fences. Natural language mixed with JSON, or arrays like\n * [{\"type\":\"Tap\"}, {\"type\":\"Hover\"}], are outside the supported contract and\n * are not reliably recoverable.\n *\n * This extractor is also used by extraction responses that can be any\n * JSON value. In those cases, arrays/strings/numbers have no object braces and\n * pass through unchanged.\n */\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport type JsonParserSource =\n  | 'generic-object'\n  | 'planning-action-param'\n  | 'locate'\n  | 'section-locator';\n\nexport interface JsonParserContext {\n  source: JsonParserSource;\n  preserveStringValueKeys?: string[];\n}\n\nexport type JsonParser = (raw: string, context?: JsonParserContext) => unknown;\n\n/**\n * Trim whitespace in parsed JSON by normalizing:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction trimParsedJsonStrings(\n  obj: any,\n  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},\n): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => trimParsedJsonStrings(item, context));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n      const preserveStringValue =\n        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;\n\n      const normalizedValue =\n        typeof value === 'string'\n          ? preserveStringValue\n            ? value\n            : value.trim()\n          : trimParsedJsonStrings(value, context);\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nfunction repairKnownJsonIssues(\n  jsonBlock: string,\n  _rawResponse: string,\n): string {\n  // TODO: Add project-specific repairs that jsonrepair cannot handle.\n  return jsonBlock;\n}\n\nexport function assertJsonObject(\n  parsed: unknown,\n): asserts parsed is Record<string, unknown> {\n  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {\n    const parsedType = Array.isArray(parsed)\n      ? 'array'\n      : parsed === null\n        ? 'null'\n        : typeof parsed;\n    throw new Error(\n      `LLM response is valid JSON but does not match the expected schema. Expected to be a JSON object, got ${parsedType}: ${JSON.stringify(parsed)}.`,\n    );\n  }\n}\n\nfunction parseJsonWithRepair(jsonStr: string) {\n  try {\n    return JSON.parse(jsonStr);\n  } catch {\n    return JSON.parse(jsonrepair(jsonStr));\n  }\n}\n\nexport function parseModelResponseJson(\n  raw: string,\n  context?: JsonParserContext,\n) {\n  const cleanJsonString = extractJSONFromCodeBlock(raw);\n\n  let parsedObj: unknown;\n\n  try {\n    parsedObj = parseJsonWithRepair(cleanJsonString);\n  } catch (e1) {\n    const code = repairKnownJsonIssues(cleanJsonString, raw);\n    if (code === cleanJsonString) {\n      throw new Error(\n        `failed to parse LLM response into JSON. Error - ${String(\n          e1,\n        )}. Response - \\n ${raw}`,\n      );\n    }\n\n    try {\n      parsedObj = parseJsonWithRepair(code);\n    } catch (e2) {\n      throw new Error(\n        `failed to parse LLM response into JSON. First error - ${String(\n          e1,\n        )}. Second error - ${String(e2)}. Response - \\n ${raw}`,\n      );\n    }\n  }\n\n  return trimParsedJsonStrings(parsedObj, context);\n}\n"],"names":["extractJSONFromCodeBlock","response","jsonMatch","codeBlockMatch","jsonLikeMatch","trimParsedJsonStrings","obj","context","Array","item","normalized","key","value","Object","trimmedKey","preserveStringValue","normalizedValue","repairKnownJsonIssues","jsonBlock","_rawResponse","assertJsonObject","parsed","parsedType","Error","JSON","parseJsonWithRepair","jsonStr","jsonrepair","parseModelResponseJson","raw","cleanJsonString","parsedObj","e1","code","String","e2"],"mappings":";AAmBO,SAASA,yBAAyBC,QAAgB;IACvD,IAAI;QAEF,MAAMC,YAAYD,SAAS,KAAK,CAAC;QACjC,IAAIC,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBF,SAAS,KAAK,CACnC;QAEF,IAAIE,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBH,SAAS,KAAK,CAAC;QACrC,IAAIG,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOH;AACT;AAqBA,SAASI,sBACPC,GAAQ,EACRC,UAA8D,CAAC,CAAC;IAGhE,IAAID,QAAAA,KACF,OAAOA;IAIT,IAAIE,MAAM,OAAO,CAACF,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACG,OAASJ,sBAAsBI,MAAMF;IAIvD,IAAI,AAAe,YAAf,OAAOD,KAAkB;QAC3B,MAAMI,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACP,KAAM;YAE9C,MAAMQ,aAAaH,IAAI,IAAI;YAC3B,MAAMI,sBACJR,QAAQ,uBAAuB,EAAE,SAASO,eAAe;YAE3D,MAAME,kBACJ,AAAiB,YAAjB,OAAOJ,QACHG,sBACEH,QACAA,MAAM,IAAI,KACZP,sBAAsBO,OAAOL;YAEnCG,UAAU,CAACI,WAAW,GAAGE;QAC3B;QAEA,OAAON;IACT;IAGA,IAAI,AAAe,YAAf,OAAOJ,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEA,SAASW,sBACPC,SAAiB,EACjBC,YAAoB;IAGpB,OAAOD;AACT;AAEO,SAASE,iBACdC,MAAe;IAEf,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,UAAuBb,MAAM,OAAO,CAACa,SAAS;QAClE,MAAMC,aAAad,MAAM,OAAO,CAACa,UAC7B,UACAA,AAAW,SAAXA,SACE,SACA,OAAOA;QACb,MAAM,IAAIE,MACR,CAAC,qGAAqG,EAAED,WAAW,EAAE,EAAEE,KAAK,SAAS,CAACH,QAAQ,CAAC,CAAC;IAEpJ;AACF;AAEA,SAASI,oBAAoBC,OAAe;IAC1C,IAAI;QACF,OAAOF,KAAK,KAAK,CAACE;IACpB,EAAE,OAAM;QACN,OAAOF,KAAK,KAAK,CAACG,WAAWD;IAC/B;AACF;AAEO,SAASE,uBACdC,GAAW,EACXtB,OAA2B;IAE3B,MAAMuB,kBAAkB9B,yBAAyB6B;IAEjD,IAAIE;IAEJ,IAAI;QACFA,YAAYN,oBAAoBK;IAClC,EAAE,OAAOE,IAAI;QACX,MAAMC,OAAOhB,sBAAsBa,iBAAiBD;QACpD,IAAII,SAASH,iBACX,MAAM,IAAIP,MACR,CAAC,gDAAgD,EAAEW,OACjDF,IACA,gBAAgB,EAAEH,KAAK;QAI7B,IAAI;YACFE,YAAYN,oBAAoBQ;QAClC,EAAE,OAAOE,IAAI;YACX,MAAM,IAAIZ,MACR,CAAC,sDAAsD,EAAEW,OACvDF,IACA,iBAAiB,EAAEE,OAAOC,IAAI,gBAAgB,EAAEN,KAAK;QAE3D;IACF;IAEA,OAAOxB,sBAAsB0B,WAAWxB;AAC1C"}
+{"version":3,"file":"ai-model/shared/json.mjs","sources":["../../../../src/ai-model/shared/json.ts"],"sourcesContent":["import { jsonrepair } from 'jsonrepair';\n\n/**\n * Preserve complete JSON values and remove optional markdown fences before\n * parsing. Keep array boundaries intact, including when the contents need repair.\n * For prose mixed with JSON, fall back to extracting the first through last\n * object brace; arbitrary prose with arrays is not a supported format.\n */\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    JSON.parse(response);\n    return response.trim();\n  } catch {\n    // Invalid JSON may still be recoverable after extraction and repair.\n  }\n\n  try {\n    // Preserve complete object and array boundaries for jsonrepair.\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\}|\\[[\\s\\S]*\\])\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(/```(?:json)?\\s*([\\s\\S]*?)\\s*```/i);\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport type JsonParserSource =\n  | 'generic-object'\n  | 'planning-action-param'\n  | 'locate'\n  | 'section-locator';\n\nexport interface JsonParserContext {\n  source: JsonParserSource;\n  preserveStringValueKeys?: string[];\n}\n\nexport type JsonParser = (raw: string, context?: JsonParserContext) => unknown;\n\n/**\n * Trim whitespace in parsed JSON by normalizing:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction trimParsedJsonStrings(\n  obj: any,\n  context: Pick<JsonParserContext, 'preserveStringValueKeys'> = {},\n): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => trimParsedJsonStrings(item, context));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n      const preserveStringValue =\n        context.preserveStringValueKeys?.includes(trimmedKey) ?? false;\n\n      const normalizedValue =\n        typeof value === 'string'\n          ? preserveStringValue\n            ? value\n            : value.trim()\n          : trimParsedJsonStrings(value, context);\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nfunction repairKnownJsonIssues(\n  jsonBlock: string,\n  _rawResponse: string,\n): string {\n  // TODO: Add project-specific repairs that jsonrepair cannot handle.\n  return jsonBlock;\n}\n\nexport function assertJsonObject(\n  parsed: unknown,\n): asserts parsed is Record<string, unknown> {\n  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {\n    const parsedType = Array.isArray(parsed)\n      ? 'array'\n      : parsed === null\n        ? 'null'\n        : typeof parsed;\n    throw new Error(\n      `LLM response is valid JSON but does not match the expected schema. Expected to be a JSON object, got ${parsedType}: ${JSON.stringify(parsed)}.`,\n    );\n  }\n}\n\nfunction parseJsonWithRepair(jsonStr: string) {\n  try {\n    return JSON.parse(jsonStr);\n  } catch {\n    return JSON.parse(jsonrepair(jsonStr));\n  }\n}\n\nexport function parseModelResponseJson(\n  raw: string,\n  context?: JsonParserContext,\n) {\n  const cleanJsonString = extractJSONFromCodeBlock(raw);\n\n  let parsedObj: unknown;\n\n  try {\n    parsedObj = parseJsonWithRepair(cleanJsonString);\n  } catch (e1) {\n    const code = repairKnownJsonIssues(cleanJsonString, raw);\n    if (code === cleanJsonString) {\n      throw new Error(\n        `failed to parse LLM response into JSON. Error - ${String(\n          e1,\n        )}. Response - \\n ${raw}`,\n      );\n    }\n\n    try {\n      parsedObj = parseJsonWithRepair(code);\n    } catch (e2) {\n      throw new Error(\n        `failed to parse LLM response into JSON. First error - ${String(\n          e1,\n        )}. Second error - ${String(e2)}. Response - \\n ${raw}`,\n      );\n    }\n  }\n\n  return trimParsedJsonStrings(parsedObj, context);\n}\n"],"names":["extractJSONFromCodeBlock","response","JSON","jsonMatch","codeBlockMatch","jsonLikeMatch","trimParsedJsonStrings","obj","context","Array","item","normalized","key","value","Object","trimmedKey","preserveStringValue","normalizedValue","repairKnownJsonIssues","jsonBlock","_rawResponse","assertJsonObject","parsed","parsedType","Error","parseJsonWithRepair","jsonStr","jsonrepair","parseModelResponseJson","raw","cleanJsonString","parsedObj","e1","code","String","e2"],"mappings":";AAQO,SAASA,yBAAyBC,QAAgB;IACvD,IAAI;QACFC,KAAK,KAAK,CAACD;QACX,OAAOA,SAAS,IAAI;IACtB,EAAE,OAAM,CAER;IAEA,IAAI;QAEF,MAAME,YAAYF,SAAS,KAAK,CAAC;QACjC,IAAIE,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBH,SAAS,KAAK,CAAC;QACtC,IAAIG,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBJ,SAAS,KAAK,CAAC;QACrC,IAAII,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOJ;AACT;AAqBA,SAASK,sBACPC,GAAQ,EACRC,UAA8D,CAAC,CAAC;IAGhE,IAAID,QAAAA,KACF,OAAOA;IAIT,IAAIE,MAAM,OAAO,CAACF,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACG,OAASJ,sBAAsBI,MAAMF;IAIvD,IAAI,AAAe,YAAf,OAAOD,KAAkB;QAC3B,MAAMI,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACP,KAAM;YAE9C,MAAMQ,aAAaH,IAAI,IAAI;YAC3B,MAAMI,sBACJR,QAAQ,uBAAuB,EAAE,SAASO,eAAe;YAE3D,MAAME,kBACJ,AAAiB,YAAjB,OAAOJ,QACHG,sBACEH,QACAA,MAAM,IAAI,KACZP,sBAAsBO,OAAOL;YAEnCG,UAAU,CAACI,WAAW,GAAGE;QAC3B;QAEA,OAAON;IACT;IAGA,IAAI,AAAe,YAAf,OAAOJ,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEA,SAASW,sBACPC,SAAiB,EACjBC,YAAoB;IAGpB,OAAOD;AACT;AAEO,SAASE,iBACdC,MAAe;IAEf,IAAI,CAACA,UAAU,AAAkB,YAAlB,OAAOA,UAAuBb,MAAM,OAAO,CAACa,SAAS;QAClE,MAAMC,aAAad,MAAM,OAAO,CAACa,UAC7B,UACAA,AAAW,SAAXA,SACE,SACA,OAAOA;QACb,MAAM,IAAIE,MACR,CAAC,qGAAqG,EAAED,WAAW,EAAE,EAAErB,KAAK,SAAS,CAACoB,QAAQ,CAAC,CAAC;IAEpJ;AACF;AAEA,SAASG,oBAAoBC,OAAe;IAC1C,IAAI;QACF,OAAOxB,KAAK,KAAK,CAACwB;IACpB,EAAE,OAAM;QACN,OAAOxB,KAAK,KAAK,CAACyB,WAAWD;IAC/B;AACF;AAEO,SAASE,uBACdC,GAAW,EACXrB,OAA2B;IAE3B,MAAMsB,kBAAkB9B,yBAAyB6B;IAEjD,IAAIE;IAEJ,IAAI;QACFA,YAAYN,oBAAoBK;IAClC,EAAE,OAAOE,IAAI;QACX,MAAMC,OAAOf,sBAAsBY,iBAAiBD;QACpD,IAAII,SAASH,iBACX,MAAM,IAAIN,MACR,CAAC,gDAAgD,EAAEU,OACjDF,IACA,gBAAgB,EAAEH,KAAK;QAI7B,IAAI;YACFE,YAAYN,oBAAoBQ;QAClC,EAAE,OAAOE,IAAI;YACX,MAAM,IAAIX,MACR,CAAC,sDAAsD,EAAEU,OACvDF,IACA,iBAAiB,EAAEE,OAAOC,IAAI,gBAAgB,EAAEN,KAAK;QAE3D;IACF;IAEA,OAAOvB,sBAAsByB,WAAWvB;AAC1C"}

package/dist/es/ai-model/workflows/grounding/locate.mjs

@@ -84,7 +84,7 @@ async function genericLocate(locateRequest) {
         locateImage: locateRequest.locateImage,
         imagePreprocess: adapter.imagePreprocess,
         targetDescription: targetElementDescription,
-        userMessageContentOrder: adapter.locate.userMessageContentOrder
+        promptOrder: adapter.locate.promptOrder
     });
     try {
         return await callAiAndParseWithRetry({

package/dist/es/ai-model/workflows/grounding/locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawAssistantOutput","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;;;;;;AAsBA,MAAMA,iBAAiBC,SAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,OACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,oCACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,mBAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,OACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,+BAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,mBAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,yBAAyByB,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAI;QACF,OAAO,MAAMS,wBAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,iBACEC,0BAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,oBAAoB4B,SAAS,kBAAkB;oBAC/C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,oBAAoB4B,SAAS,kBAAkB;wBAC/C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,yBAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,qBACTD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/locate.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport { buildElementLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mapSearchAreaResultToOriginalResult } from './search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequest,\n  LocateResult,\n} from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugGrounding = getDebug('ai:grounding');\n\nexport { userPromptToString as extraTextFromUserPrompt } from '../../../common';\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const locateRequest: LocateRequest = {\n    targetElementDescription,\n    locateImage,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(locateRequest);\n  const {\n    locatedPixelResult,\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawAssistantOutput,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelResult) {\n    return {\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const mappedResult = mapSearchAreaResultToOriginalResult(\n      locatedPixelResult,\n      locateOptions.searchConfig?.mapping,\n    );\n    debugGrounding('center', mappedResult.center);\n\n    return {\n      parseResult: {\n        element: {\n          ...mappedResult,\n          description: userPromptToString(targetElementDescription),\n        },\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  locateRequest: LocateRequest,\n): Promise<LocateModelResponse> {\n  const { options, targetElementDescription } = locateRequest;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const { protocol, resultCodec } = adapter.locate.element;\n\n  const systemPrompt = buildElementLocateSystemPrompt({\n    systemPromptIntroduction: protocol.systemPromptIntroduction,\n    responseInstructions: protocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = protocol.buildUserPrompt(\n    userPromptToString(targetElementDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: locateRequest.locateImage,\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: targetElementDescription,\n    promptOrder: adapter.locate.promptOrder,\n  });\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse: protocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response): LocateModelResponse => {\n        const parsedLocateResult = protocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const rawResponse = response.content;\n        const locateError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return {\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: locateError ? [locateError] : [],\n          };\n        }\n\n        try {\n          const result = resultCodec.toPixelResult(parsedLocateResult.target, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n          return {\n            locatedPixelResult: result,\n            rawResponse,\n            rawAssistantOutput: response.rawAssistantOutput,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: [],\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            locateError && `error in locate result: ${locateError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugGrounding(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n"],"names":["debugGrounding","getDebug","AiLocateElement","options","targetElementDescription","locateOptions","assert","context","locateImage","locateRequest","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelResult","rawResponse","rawAssistantOutput","usage","reasoningContent","errors","baseLocateResult","undefined","mappedResult","mapSearchAreaResultToOriginalResult","userPromptToString","error","msg","Error","modelRuntime","adapter","protocol","resultCodec","systemPrompt","buildElementLocateSystemPrompt","userInstructionPrompt","messages","preparedImage","prepareLocateModelInput","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","locateError","result","errorMessage","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError"],"mappings":";;;;;;;;AAsBA,MAAMA,iBAAiBC,SAAS;AAIzB,eAAeC,gBACpBC,OAAkE;IAElE,MAAM,EAAEC,wBAAwB,EAAE,GAAGC,eAAe,GAAGF;IACvDG,OACEF,0BACA;IAGF,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMG,cAAcH,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaE,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAME,gBAA+B;QACnCL;QACAI;QACA,SAASH;IACX;IAEA,MAAMK,gBAAgBP,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMQ,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAASF;IACtC,MAAM,EACJK,kBAAkB,EAClBC,WAAW,EACXC,kBAAkB,EAClBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,oBACH,OAAO;QACL,aAAa;YACX,SAASO;YACTF;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAME,eAAeC,oCACnBT,oBACAT,cAAc,YAAY,EAAE;QAE9BL,eAAe,UAAUsB,aAAa,MAAM;QAE5C,OAAO;YACL,aAAa;gBACX,SAAS;oBACP,GAAGA,YAAY;oBACf,aAAaE,mBAAmBpB;gBAClC;gBACA,QAAQ,EAAE;YACZ;YACA,GAAGgB,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,aAAa;gBACX,SAASJ;gBACT,QAAQF,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBH,aAA4B;IAE5B,MAAM,EAAEN,OAAO,EAAEC,wBAAwB,EAAE,GAAGK;IAC9C,MAAMmB,eAAezB,QAAQ,YAAY;IACzC,MAAM,EAAE0B,OAAO,EAAE,GAAGD;IACpBtB,OACEuB,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAM,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGF,QAAQ,MAAM,CAAC,OAAO;IAExD,MAAMG,eAAeC,+BAA+B;QAClD,0BAA0BH,SAAS,wBAAwB;QAC3D,sBAAsBA,SAAS,yBAAyB,CACtDC,YAAY,UAAU;IAE1B;IAEA,MAAMG,wBAAwBJ,SAAS,eAAe,CACpDN,mBAAmBpB;IAGrB,MAAM,EAAE+B,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEL;QACA,YAAYE;QACZ,aAAazB,cAAc,WAAW;QACtC,iBAAiBoB,QAAQ,eAAe;QACxC,mBAAmBzB;QACnB,aAAayB,QAAQ,MAAM,CAAC,WAAW;IACzC;IAEA,IAAI;QACF,OAAO,MAAMS,wBAAwB;YACnC,QAAQ,CAACC,cAAcC,qBACrBC,iBACEC,0BAA0BP,UAAUK,qBACpCZ,cACA;oBACE,aAAazB,QAAQ,WAAW;oBAChC,4BAA4B2B,SAAS,0BAA0B;oBAC/D,sBAAsBS;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBd,SAAS,gBAAgB,CAClDa,SAAS,OAAO,EAChBZ,YAAY,UAAU;gBAExB,MAAMhB,cAAc4B,SAAS,OAAO;gBACpC,MAAME,cAAcD,mBAAmB,KAAK;gBAC5C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBACL7B;oBACA,oBAAoB4B,SAAS,kBAAkB;oBAC/C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQE,cAAc;wBAACA;qBAAY,GAAG,EAAE;gBAC1C;gBAGF,IAAI;oBACF,MAAMC,SAASf,YAAY,aAAa,CAACa,mBAAmB,MAAM,EAAE;wBAClE,cAAcR,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,OAAO;wBACL,oBAAoBU;wBACpB/B;wBACA,oBAAoB4B,SAAS,kBAAkB;wBAC/C,OAAOA,SAAS,KAAK;wBACrB,kBAAkBA,SAAS,iBAAiB;wBAC5C,QAAQ,EAAE;oBACZ;gBACF,EAAE,OAAOlB,OAAO;oBACd,MAAMsB,eACJtB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;oBAClD,MAAMwB,UAAU;wBACdJ,eAAe,CAAC,wBAAwB,EAAEA,aAAa;wBACvD,CAAC,0BAA0B,EAAEE,cAAc;wBAC3CG,yBAAyBtB;qBAC1B,CACE,MAAM,CAACuB,SACP,IAAI,CAAC;oBACR,MAAM,IAAIxB,MAAMsB,SAAS;wBAAE,OAAOxB;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOkB;gBACpB,MAAMS,oBACJ3B,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,OAAO,IAAI4B,qBACTD,mBACAT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAazB,QAAQ,WAAW;YAChC,cAAc,CAACsB;gBACbzB,eACE,uDACAyB,iBAAiBE,QAAQF,MAAM,OAAO,GAAGuB,OAAOvB;YAEpD;QACF;IACF,EAAE,OAAO6B,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,aAAaC,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMP,eACJO,qBAAqB3B,QAAQ2B,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,aAAaP;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/section-locate.mjs

@@ -31,7 +31,7 @@ async function AiLocateSection(options) {
         },
         imagePreprocess: adapter.imagePreprocess,
         targetDescription: sectionDescription,
-        userMessageContentOrder: adapter.locate.userMessageContentOrder
+        promptOrder: adapter.locate.promptOrder
     });
     let parsedResult;
     try {

package/dist/es/ai-model/workflows/grounding/section-locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/section-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callGenerativeAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;;;;;;AAwBA,MAAMA,eAAeC,SAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,OAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,kCAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,mBAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,wBAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,iBACEC,0BAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,uBAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,yBAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,qBACTD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,oBAAoBA,OAAO,kBAAkB;QAC7C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,iBAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,sBAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/section-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["import type { RawAssistantOutput } from '@/types';\nimport type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport {\n  AIResponseParseError,\n  callGenerativeAI,\n} from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callGenerativeAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawAssistantOutput?: RawAssistantOutput;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    promptOrder: adapter.locate.promptOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callGenerativeAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawAssistantOutput,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawAssistantOutput: callError.rawAssistantOutput,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawAssistantOutput: result.rawAssistantOutput,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callGenerativeAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;;;;;;AAwBA,MAAMA,eAAeC,SAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,OAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,kCAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,mBAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,aAAaE,QAAQ,MAAM,CAAC,WAAW;IACzC;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,wBAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,iBACEC,0BAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,uBAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,yBAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,qBACTD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,oBAAoBA,UAAU,kBAAkB;YAChD,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,oBAAoBA,OAAO,kBAAkB;QAC7C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,iBAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,sBAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,oBAAoBA,OAAO,kBAAkB;YAC7C,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/utils.mjs

@@ -1,7 +1,8 @@
 import { userPromptToMultimodalPrompt } from "../../../common.mjs";
 import { prepareModelImage } from "../../model-adapter/image-preprocess.mjs";
+import { buildLocateMessages } from "../../model-adapter/locate-prompt-order.mjs";
 import { multimodalPromptToMessages } from "../../shared/multimodal-prompt.mjs";
-async function prepareLocateModelInput({ locateImage, targetDescription, systemPrompt, userPrompt, imagePreprocess, userMessageContentOrder }) {
+async function prepareLocateModelInput({ locateImage, targetDescription, systemPrompt, userPrompt, imagePreprocess, promptOrder }) {
     const preparedImage = await prepareModelImage({
         imageBase64: locateImage.imageBase64,
         width: locateImage.width,
@@ -9,33 +10,14 @@ async function prepareLocateModelInput({ locateImage, targetDescription, systemP
         policy: imagePreprocess
     });
     const referenceImageMessages = await multimodalPromptToMessages(userPromptToMultimodalPrompt(targetDescription));
-    const imageContent = {
-        type: 'image',
-        url: preparedImage.imageBase64
-    };
-    const promptContent = {
-        type: 'text',
-        text: userPrompt
-    };
     return {
         preparedImage,
-        messages: [
-            {
-                role: 'system',
-                content: systemPrompt
-            },
-            {
-                role: 'user',
-                content: 'prompt-first' === userMessageContentOrder ? [
-                    promptContent,
-                    imageContent
-                ] : [
-                    imageContent,
-                    promptContent
-                ]
-            },
-            ...referenceImageMessages
-        ]
+        messages: buildLocateMessages(promptOrder, {
+            instructions: systemPrompt,
+            screenshot: preparedImage.imageBase64,
+            task: userPrompt,
+            references: referenceImageMessages
+        })
     };
 }
 function formatLocateModelContext(modelRuntime) {

package/dist/es/ai-model/workflows/grounding/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/utils.mjs","sources":["../../../../../src/ai-model/workflows/grounding/utils.ts"],"sourcesContent":["import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n  MessageContent,\n} from '@/ai-model/service-caller/types';\nimport { type TUserPrompt, userPromptToMultimodalPrompt } from '@/common';\nimport {\n  type ImagePreprocessPolicy,\n  type PreparedModelImage,\n  prepareModelImage,\n} from '../../model-adapter/image-preprocess';\nimport type { LocateUserMessageContentOrder } from '../../model-adapter/types';\nimport type { ModelRuntime } from '../../models';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\n\nexport type GroundingAIArgs = [\n  ConversationSystemMessage,\n  ...ConversationUserMessage[],\n];\n\nexport async function prepareLocateModelInput({\n  locateImage,\n  targetDescription,\n  systemPrompt,\n  userPrompt,\n  imagePreprocess,\n  userMessageContentOrder,\n}: {\n  locateImage: {\n    imageBase64: string;\n    width: number;\n    height: number;\n  };\n  targetDescription: TUserPrompt;\n  systemPrompt: string;\n  userPrompt: string;\n  imagePreprocess: ImagePreprocessPolicy;\n  userMessageContentOrder: LocateUserMessageContentOrder;\n}): Promise<{\n  messages: GroundingAIArgs;\n  preparedImage: PreparedModelImage;\n}> {\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateImage.imageBase64,\n    width: locateImage.width,\n    height: locateImage.height,\n    policy: imagePreprocess,\n  });\n  const referenceImageMessages = await multimodalPromptToMessages(\n    userPromptToMultimodalPrompt(targetDescription),\n  );\n  const imageContent: MessageContent = {\n    type: 'image',\n    url: preparedImage.imageBase64,\n  };\n  const promptContent: MessageContent = {\n    type: 'text',\n    text: userPrompt,\n  };\n\n  return {\n    preparedImage,\n    messages: [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content:\n          userMessageContentOrder === 'prompt-first'\n            ? [promptContent, imageContent]\n            : [imageContent, promptContent],\n      },\n      ...referenceImageMessages,\n    ],\n  };\n}\n\nexport function formatLocateModelContext(modelRuntime: ModelRuntime): string {\n  const { modelFamily, modelName } = modelRuntime.config;\n  return `modelName=${modelName ?? 'unset'} modelFamily=${modelFamily ?? 'unset'}`;\n}\n"],"names":["prepareLocateModelInput","locateImage","targetDescription","systemPrompt","userPrompt","imagePreprocess","userMessageContentOrder","preparedImage","prepareModelImage","referenceImageMessages","multimodalPromptToMessages","userPromptToMultimodalPrompt","imageContent","promptContent","formatLocateModelContext","modelRuntime","modelFamily","modelName"],"mappings":";;;AAoBO,eAAeA,wBAAwB,EAC5CC,WAAW,EACXC,iBAAiB,EACjBC,YAAY,EACZC,UAAU,EACVC,eAAe,EACfC,uBAAuB,EAYxB;IAIC,MAAMC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,YAAY,WAAW;QACpC,OAAOA,YAAY,KAAK;QACxB,QAAQA,YAAY,MAAM;QAC1B,QAAQI;IACV;IACA,MAAMI,yBAAyB,MAAMC,2BACnCC,6BAA6BT;IAE/B,MAAMU,eAA+B;QACnC,MAAM;QACN,KAAKL,cAAc,WAAW;IAChC;IACA,MAAMM,gBAAgC;QACpC,MAAM;QACN,MAAMT;IACR;IAEA,OAAO;QACLG;QACA,UAAU;YACR;gBAAE,MAAM;gBAAU,SAASJ;YAAa;YACxC;gBACE,MAAM;gBACN,SACEG,AAA4B,mBAA5BA,0BACI;oBAACO;oBAAeD;iBAAa,GAC7B;oBAACA;oBAAcC;iBAAc;YACrC;eACGJ;SACJ;IACH;AACF;AAEO,SAASK,yBAAyBC,YAA0B;IACjE,MAAM,EAAEC,WAAW,EAAEC,SAAS,EAAE,GAAGF,aAAa,MAAM;IACtD,OAAO,CAAC,UAAU,EAAEE,aAAa,QAAQ,aAAa,EAAED,eAAe,SAAS;AAClF"}
+{"version":3,"file":"ai-model/workflows/grounding/utils.mjs","sources":["../../../../../src/ai-model/workflows/grounding/utils.ts"],"sourcesContent":["import { type TUserPrompt, userPromptToMultimodalPrompt } from '@/common';\nimport {\n  type ImagePreprocessPolicy,\n  type PreparedModelImage,\n  prepareModelImage,\n} from '../../model-adapter/image-preprocess';\nimport {\n  type LocateMessages,\n  type LocatePromptOrder,\n  buildLocateMessages,\n} from '../../model-adapter/locate-prompt-order';\nimport type { ModelRuntime } from '../../models';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\n\nexport async function prepareLocateModelInput({\n  locateImage,\n  targetDescription,\n  systemPrompt,\n  userPrompt,\n  imagePreprocess,\n  promptOrder,\n}: {\n  locateImage: {\n    imageBase64: string;\n    width: number;\n    height: number;\n  };\n  targetDescription: TUserPrompt;\n  systemPrompt: string;\n  userPrompt: string;\n  imagePreprocess: ImagePreprocessPolicy;\n  promptOrder: LocatePromptOrder;\n}): Promise<{\n  messages: LocateMessages;\n  preparedImage: PreparedModelImage;\n}> {\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateImage.imageBase64,\n    width: locateImage.width,\n    height: locateImage.height,\n    policy: imagePreprocess,\n  });\n  const referenceImageMessages = await multimodalPromptToMessages(\n    userPromptToMultimodalPrompt(targetDescription),\n  );\n  return {\n    preparedImage,\n    messages: buildLocateMessages(promptOrder, {\n      instructions: systemPrompt,\n      screenshot: preparedImage.imageBase64,\n      task: userPrompt,\n      references: referenceImageMessages,\n    }),\n  };\n}\n\nexport function formatLocateModelContext(modelRuntime: ModelRuntime): string {\n  const { modelFamily, modelName } = modelRuntime.config;\n  return `modelName=${modelName ?? 'unset'} modelFamily=${modelFamily ?? 'unset'}`;\n}\n"],"names":["prepareLocateModelInput","locateImage","targetDescription","systemPrompt","userPrompt","imagePreprocess","promptOrder","preparedImage","prepareModelImage","referenceImageMessages","multimodalPromptToMessages","userPromptToMultimodalPrompt","buildLocateMessages","formatLocateModelContext","modelRuntime","modelFamily","modelName"],"mappings":";;;;AAcO,eAAeA,wBAAwB,EAC5CC,WAAW,EACXC,iBAAiB,EACjBC,YAAY,EACZC,UAAU,EACVC,eAAe,EACfC,WAAW,EAYZ;IAIC,MAAMC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,YAAY,WAAW;QACpC,OAAOA,YAAY,KAAK;QACxB,QAAQA,YAAY,MAAM;QAC1B,QAAQI;IACV;IACA,MAAMI,yBAAyB,MAAMC,2BACnCC,6BAA6BT;IAE/B,OAAO;QACLK;QACA,UAAUK,oBAAoBN,aAAa;YACzC,cAAcH;YACd,YAAYI,cAAc,WAAW;YACrC,MAAMH;YACN,YAAYK;QACd;IACF;AACF;AAEO,SAASI,yBAAyBC,YAA0B;IACjE,MAAM,EAAEC,WAAW,EAAEC,SAAS,EAAE,GAAGF,aAAa,MAAM;IACtD,OAAO,CAAC,UAAU,EAAEE,aAAa,QAAQ,aAAa,EAAED,eAAe,SAAS;AAClF"}

package/dist/es/dump/report-image-export.mjs

@@ -0,0 +1,58 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { imageRefFileExtension } from "./image-reference.mjs";
+import { parseBase64ImageDataUrl, reportImageFileName, resolveImageSource } from "./screenshot-store.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class ReportImageExporter {
+    write(ref, dataUri) {
+        const key = JSON.stringify([
+            ref.id,
+            ref.mimeType,
+            ref.storage,
+            ref.path
+        ]);
+        const known = this.locations.get(key);
+        if (known) return known;
+        const source = dataUri ? {
+            type: 'data-uri',
+            dataUri
+        } : resolveImageSource(ref, {
+            reportPath: this.reportPath
+        });
+        const bytes = 'file' === source.type ? readFileSync(source.filePath) : Buffer.from(parseBase64ImageDataUrl(source.dataUri).rawBase64, 'base64');
+        const name = reportImageFileName(imageRefFileExtension(ref), bytes);
+        if (!this.writtenFiles.has(name)) {
+            writeFileSync(join(this.screenshotsDir, name), bytes);
+            this.writtenFiles.add(name);
+        }
+        this.locations.set(key, name);
+        return name;
+    }
+    get files() {
+        return [
+            ...this.writtenFiles
+        ].sort().map((name)=>join(this.screenshotsDir, name));
+    }
+    constructor(reportPath, screenshotsDir){
+        _define_property(this, "reportPath", void 0);
+        _define_property(this, "screenshotsDir", void 0);
+        _define_property(this, "locations", void 0);
+        _define_property(this, "writtenFiles", void 0);
+        this.reportPath = reportPath;
+        this.screenshotsDir = screenshotsDir;
+        this.locations = new Map();
+        this.writtenFiles = new Set();
+    }
+}
+export { ReportImageExporter };
+
+//# sourceMappingURL=report-image-export.mjs.map

package/dist/es/dump/report-image-export.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"dump/report-image-export.mjs","sources":["../../../src/dump/report-image-export.ts"],"sourcesContent":["import { readFileSync, writeFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { type StoredImageRef, imageRefFileExtension } from './image-reference';\nimport {\n  parseBase64ImageDataUrl,\n  reportImageFileName,\n  resolveImageSource,\n} from './screenshot-store';\n\n/** Export image bytes once while keeping every observation's own metadata. */\nexport class ReportImageExporter {\n  private readonly locations = new Map<string, string>();\n  private readonly writtenFiles = new Set<string>();\n\n  constructor(\n    private readonly reportPath: string,\n    private readonly screenshotsDir: string,\n  ) {}\n\n  write(ref: StoredImageRef, dataUri?: string): string {\n    const key = JSON.stringify([ref.id, ref.mimeType, ref.storage, ref.path]);\n    const known = this.locations.get(key);\n    if (known) return known;\n    const source = dataUri\n      ? { type: 'data-uri' as const, dataUri }\n      : resolveImageSource(ref, { reportPath: this.reportPath });\n    const bytes =\n      source.type === 'file'\n        ? readFileSync(source.filePath)\n        : Buffer.from(\n            parseBase64ImageDataUrl(source.dataUri).rawBase64,\n            'base64',\n          );\n    const name = reportImageFileName(imageRefFileExtension(ref), bytes);\n    if (!this.writtenFiles.has(name)) {\n      writeFileSync(join(this.screenshotsDir, name), bytes);\n      this.writtenFiles.add(name);\n    }\n    this.locations.set(key, name);\n    return name;\n  }\n\n  get files(): string[] {\n    return [...this.writtenFiles]\n      .sort()\n      .map((name) => join(this.screenshotsDir, name));\n  }\n}\n"],"names":["ReportImageExporter","ref","dataUri","key","JSON","known","source","resolveImageSource","bytes","readFileSync","Buffer","parseBase64ImageDataUrl","name","reportImageFileName","imageRefFileExtension","writeFileSync","join","reportPath","screenshotsDir","Map","Set"],"mappings":";;;;;;;;;;;;;;AAUO,MAAMA;IASX,MAAMC,GAAmB,EAAEC,OAAgB,EAAU;QACnD,MAAMC,MAAMC,KAAK,SAAS,CAAC;YAACH,IAAI,EAAE;YAAEA,IAAI,QAAQ;YAAEA,IAAI,OAAO;YAAEA,IAAI,IAAI;SAAC;QACxE,MAAMI,QAAQ,IAAI,CAAC,SAAS,CAAC,GAAG,CAACF;QACjC,IAAIE,OAAO,OAAOA;QAClB,MAAMC,SAASJ,UACX;YAAE,MAAM;YAAqBA;QAAQ,IACrCK,mBAAmBN,KAAK;YAAE,YAAY,IAAI,CAAC,UAAU;QAAC;QAC1D,MAAMO,QACJF,AAAgB,WAAhBA,OAAO,IAAI,GACPG,aAAaH,OAAO,QAAQ,IAC5BI,OAAO,IAAI,CACTC,wBAAwBL,OAAO,OAAO,EAAE,SAAS,EACjD;QAER,MAAMM,OAAOC,oBAAoBC,sBAAsBb,MAAMO;QAC7D,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAACI,OAAO;YAChCG,cAAcC,KAAK,IAAI,CAAC,cAAc,EAAEJ,OAAOJ;YAC/C,IAAI,CAAC,YAAY,CAAC,GAAG,CAACI;QACxB;QACA,IAAI,CAAC,SAAS,CAAC,GAAG,CAACT,KAAKS;QACxB,OAAOA;IACT;IAEA,IAAI,QAAkB;QACpB,OAAO;eAAI,IAAI,CAAC,YAAY;SAAC,CAC1B,IAAI,GACJ,GAAG,CAAC,CAACA,OAASI,KAAK,IAAI,CAAC,cAAc,EAAEJ;IAC7C;IAhCA,YACmBK,UAAkB,EAClBC,cAAsB,CACvC;;;QANF,uBAAiB,aAAjB;QACA,uBAAiB,gBAAjB;aAGmBD,UAAU,GAAVA;aACAC,cAAc,GAAdA;aALF,SAAS,GAAG,IAAIC;aAChB,YAAY,GAAG,IAAIC;IAKjC;AA8BL"}

package/dist/es/dump/screenshot-store.mjs

@@ -87,6 +87,11 @@ class ReportImageStore {
         if (!file) throw new Error(`ReportImageStore: image ${id} has no persisted file`);
         return file.relativePath;
     }
+    persistFileCopy(screenshot) {
+        return this.persistToSharedFileIfNeeded(screenshot, {
+            markAsPersisted: false
+        });
+    }
     async persist(screenshot) {
         await this.ensureExistingInlineImageIdsLoaded();
         const shouldWriteFileCopy = 'directory' === this.mode || this.alsoWriteFileCopy;

package/dist/es/dump/screenshot-store.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"dump/screenshot-store.mjs","sources":["../../../src/dump/screenshot-store.ts"],"sourcesContent":["import { createHash } from 'node:crypto';\nimport { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';\nimport { dirname, isAbsolute, join } from 'node:path';\nimport type { ScreenshotItem } from '../screenshot-item';\nimport { collectImageScriptIds, extractImageByIdSync } from './html-utils';\nimport {\n  type ImageUrlRef,\n  type ScreenshotRef,\n  type StoredImageRef,\n  imageFileExtensionForMimeType,\n  imageRefFileExtension,\n  normalizeImageUrlRef,\n  normalizeScreenshotRef,\n  normalizeStoredImageRef,\n} from './image-reference';\nimport { writeReportArtifactFile } from './report-artifact-store';\n\nexport type {\n  ImageUrlRef,\n  ScreenshotRef,\n  StoredImageRef,\n} from './image-reference';\nexport {\n  imageRefFileExtension,\n  normalizeImageUrlRef,\n  normalizeScreenshotRef,\n  normalizeStoredImageRef,\n} from './image-reference';\n\nconst base64ImageDataUrlPattern =\n  /^data:(image\\/[a-z0-9.+-]+)(?:;[^,]*)?;base64,([a-z0-9+/=\\s]+)$/i;\ntype ResolvedImageSource =\n  | {\n      type: 'data-uri';\n      id: string;\n      mimeType: StoredImageRef['mimeType'];\n      dataUri: string;\n    }\n  | {\n      type: 'file';\n      id: string;\n      mimeType: StoredImageRef['mimeType'];\n      filePath: string;\n    };\n\n/** Parse and normalize a base64 image data URL for report persistence. */\nexport function parseBase64ImageDataUrl(imageUrl: string): {\n  mimeType: `image/${string}`;\n  extension: string;\n  rawBase64: string;\n} {\n  const match = imageUrl.match(base64ImageDataUrlPattern);\n  if (!match) {\n    throw new Error(\n      'ReportImageStore: reference image must be a valid base64 image data URL',\n    );\n  }\n  const mimeType = match[1].toLowerCase() as `image/${string}`;\n  return {\n    mimeType,\n    extension: imageFileExtensionForMimeType(mimeType),\n    rawBase64: match[2].replace(/\\s/g, ''),\n  };\n}\n\nexport function isBase64ImageDataUrl(value: string): boolean {\n  return base64ImageDataUrlPattern.test(value);\n}\n\n/** Stable asset names allow different observations to share the same bytes. */\nexport function reportImageFileName(\n  extension: string,\n  bytes: Uint8Array,\n): string {\n  const hash = createHash('sha256')\n    .update(extension)\n    .update('\\0')\n    .update(bytes)\n    .digest('hex');\n  return `${hash}.${extension}`;\n}\n\nexport function resolveImageSource(\n  refInput: unknown,\n  options: {\n    reportPath: string;\n    imageDirectory?: string;\n    fallbackId?: string;\n    fallbackMimeType?: ScreenshotRef['mimeType'];\n  },\n): ResolvedImageSource {\n  const ref = normalizeStoredImageRef(refInput);\n  const id = ref?.id ?? options.fallbackId;\n  const mimeType = ref?.mimeType ?? options.fallbackMimeType;\n\n  if (!id || !mimeType) {\n    throw new Error(\n      'ReportImageStore: image id and mimeType are required to resolve an image',\n    );\n  }\n\n  const resolveReportRelativePath = (filePath: string): string =>\n    isAbsolute(filePath)\n      ? filePath\n      : join(dirname(options.reportPath), filePath);\n\n  if (ref?.storage === 'file') {\n    if (!ref.path) {\n      throw new Error(\n        `ReportImageStore: image ref \"${ref.id}\" missing file path`,\n      );\n    }\n\n    const explicitFilePath = resolveReportRelativePath(ref.path);\n    if (existsSync(explicitFilePath)) {\n      return {\n        type: 'file',\n        id,\n        mimeType,\n        filePath: explicitFilePath,\n      };\n    }\n  }\n\n  const inlineDataUri = extractImageByIdSync(options.reportPath, id);\n  if (inlineDataUri) {\n    return {\n      type: 'data-uri',\n      id,\n      mimeType,\n      dataUri: inlineDataUri,\n    };\n  }\n\n  const extension = ref\n    ? imageRefFileExtension(ref)\n    : imageFileExtensionForMimeType(options.fallbackMimeType!);\n  if (options.imageDirectory) {\n    const companionImagePath = join(\n      options.imageDirectory,\n      `${id}.${extension}`,\n    );\n    if (existsSync(companionImagePath)) {\n      return {\n        type: 'file',\n        id,\n        mimeType,\n        filePath: companionImagePath,\n      };\n    }\n  }\n  const siblingImagePath = join(\n    dirname(options.reportPath),\n    'screenshots',\n    `${id}.${extension}`,\n  );\n  if (existsSync(siblingImagePath)) {\n    return {\n      type: 'file',\n      id,\n      mimeType,\n      filePath: siblingImagePath,\n    };\n  }\n\n  throw new Error(\n    `ReportImageStore: cannot resolve image \"${id}\" from ${options.reportPath}`,\n  );\n}\n\nexport function resolveScreenshotSource(\n  refInput: unknown,\n  options: {\n    reportPath: string;\n    imageDirectory?: string;\n    fallbackId?: string;\n    fallbackMimeType?: ScreenshotRef['mimeType'];\n  },\n): ResolvedImageSource {\n  const ref = normalizeScreenshotRef(refInput);\n  return resolveImageSource(ref, options);\n}\n\n/** Persists and restores every image asset referenced by a report. */\nexport class ReportImageStore {\n  private readonly mode: 'inline' | 'directory';\n  private readonly reportPath: string;\n  private readonly screenshotsDir?: string;\n  private readonly writeInlineImage?: (\n    id: string,\n    base64: string,\n  ) => void | Promise<void>;\n  private alsoWriteFileCopy: boolean;\n  private readonly writtenInlineIds = new Set<string>();\n  private readonly writtenFileIds = new Set<string>();\n  private readonly fileLocations = new Map<\n    string,\n    { relativePath: string; absolutePath: string }\n  >();\n  private readonly pendingFiles = new Map<\n    string,\n    Promise<{ relativePath: string; absolutePath: string }>\n  >();\n  private readonly shouldReuseExistingInlineReport: boolean;\n  private existingInlineImageIdsPromise?: Promise<void>;\n\n  constructor(options: {\n    mode: 'inline' | 'directory';\n    reportPath: string;\n    screenshotsDir?: string;\n    writeInlineImage?: (id: string, base64: string) => void | Promise<void>;\n    alsoWriteFileCopy?: boolean;\n    /** @deprecated Use alsoWriteFileCopy instead. */\n    ensureFileCopy?: boolean;\n    /** Reuse inline image assets already present in reportPath. */\n    reuseExistingReport?: boolean;\n  }) {\n    this.mode = options.mode;\n    this.reportPath = options.reportPath;\n    this.screenshotsDir = options.screenshotsDir;\n    this.writeInlineImage = options.writeInlineImage;\n    this.alsoWriteFileCopy =\n      options.alsoWriteFileCopy ?? options.ensureFileCopy ?? false;\n    this.shouldReuseExistingInlineReport = Boolean(\n      options.reuseExistingReport && this.mode === 'inline',\n    );\n  }\n\n  enableFileCopies(): void {\n    this.alsoWriteFileCopy = true;\n  }\n\n  /** Resolve an observation/reference ID to its content-addressed asset. */\n  getFilePath(id: string): string {\n    const file = this.fileLocations.get(id);\n    if (!file)\n      throw new Error(`ReportImageStore: image ${id} has no persisted file`);\n    return file.relativePath;\n  }\n\n  async persist(screenshot: ScreenshotItem): Promise<ScreenshotRef> {\n    await this.ensureExistingInlineImageIdsLoaded();\n    const shouldWriteFileCopy =\n      this.mode === 'directory' || this.alsoWriteFileCopy;\n    const fileRef = shouldWriteFileCopy\n      ? await this.persistToSharedFileIfNeeded(screenshot, {\n          markAsPersisted: this.mode === 'directory',\n        })\n      : null;\n\n    if (this.mode === 'inline') {\n      if (!this.writeInlineImage) {\n        throw new Error(\n          'ReportImageStore: writeInlineImage is required in inline mode',\n        );\n      }\n      if (!this.writtenInlineIds.has(screenshot.id)) {\n        await this.writeInlineImage(screenshot.id, screenshot.base64);\n        this.writtenInlineIds.add(screenshot.id);\n      }\n      return screenshot.markPersistedInline(this.reportPath);\n    }\n\n    if (!fileRef) {\n      throw new Error(\n        'ReportImageStore: file persistence is required in directory mode',\n      );\n    }\n    return fileRef;\n  }\n\n  /** Persist a base64 multimodal prompt image and return its content-addressed ref. */\n  async persistReferenceImage(imageUrl: string): Promise<ImageUrlRef> {\n    await this.ensureExistingInlineImageIdsLoaded();\n    const { mimeType, extension, rawBase64 } =\n      parseBase64ImageDataUrl(imageUrl);\n    const id = `reference-${createHash('sha256')\n      .update(mimeType)\n      .update('\\0')\n      .update(Buffer.from(rawBase64, 'base64'))\n      .digest('hex')}`;\n    const shouldWriteFileCopy =\n      this.mode === 'directory' || this.alsoWriteFileCopy;\n    const fileLocation = shouldWriteFileCopy\n      ? await this.writeImageFileIfNeeded({ id, extension, rawBase64 })\n      : null;\n\n    let ref: ImageUrlRef;\n    if (this.mode === 'inline') {\n      if (!this.writeInlineImage) {\n        throw new Error(\n          'ReportImageStore: writeInlineImage is required in inline mode',\n        );\n      }\n      if (!this.writtenInlineIds.has(id)) {\n        await this.writeInlineImage(id, imageUrl);\n        this.writtenInlineIds.add(id);\n      }\n      ref = {\n        type: 'midscene_image_url_ref',\n        id,\n        mimeType,\n        storage: 'inline',\n      };\n    } else {\n      if (!fileLocation) {\n        throw new Error(\n          'ReportImageStore: file persistence is required in directory mode',\n        );\n      }\n      ref = {\n        type: 'midscene_image_url_ref',\n        id,\n        mimeType,\n        storage: 'file',\n        path: fileLocation.relativePath,\n      };\n    }\n    return ref;\n  }\n\n  private async ensureExistingInlineImageIdsLoaded(): Promise<void> {\n    if (!this.shouldReuseExistingInlineReport) return;\n    if (!this.existingInlineImageIdsPromise) {\n      this.existingInlineImageIdsPromise = (async () => {\n        try {\n          const imageIds = await collectImageScriptIds(this.reportPath);\n          for (const imageId of imageIds) this.writtenInlineIds.add(imageId);\n        } catch (error) {\n          if (\n            typeof error === 'object' &&\n            error !== null &&\n            'code' in error &&\n            error.code === 'ENOENT'\n          ) {\n            return;\n          }\n          throw new Error(\n            `ReportImageStore: failed to index existing inline images from ${this.reportPath}`,\n            { cause: error },\n          );\n        }\n      })();\n    }\n    await this.existingInlineImageIdsPromise;\n  }\n\n  private async persistToSharedFileIfNeeded(\n    screenshot: ScreenshotItem,\n    options: {\n      markAsPersisted: boolean;\n    },\n  ): Promise<ScreenshotRef> {\n    const { relativePath, absolutePath } =\n      this.fileLocations.get(screenshot.id) ??\n      (await this.writeImageFileIfNeeded({\n        id: screenshot.id,\n        extension: screenshot.extension,\n        rawBase64: screenshot.rawBase64,\n      }));\n\n    if (options.markAsPersisted) {\n      return screenshot.markPersistedToPath(relativePath, absolutePath);\n    }\n\n    return screenshot.registerPersistedFileCopy(relativePath, absolutePath);\n  }\n\n  private async writeImageFileIfNeeded(image: {\n    id: string;\n    extension: string;\n    rawBase64: string;\n  }): Promise<{ relativePath: string; absolutePath: string }> {\n    const screenshotsDir = this.screenshotsDir;\n    if (!screenshotsDir) {\n      throw new Error(\n        'ReportImageStore: screenshotsDir is required when file persistence is enabled',\n      );\n    }\n    if (!existsSync(screenshotsDir)) {\n      mkdirSync(screenshotsDir, { recursive: true });\n    }\n\n    const known = this.fileLocations.get(image.id);\n    if (known) return known;\n    const bytes = Buffer.from(image.rawBase64, 'base64');\n    const fileName = reportImageFileName(image.extension, bytes);\n    const relativePath = `./screenshots/${fileName}`;\n    const absolutePath = join(screenshotsDir, fileName);\n    let publication = this.pendingFiles.get(fileName);\n    if (!publication) {\n      publication = (async () => {\n        if (!this.writtenFileIds.has(fileName)) {\n          await writeReportArtifactFile(absolutePath, bytes);\n          this.writtenFileIds.add(fileName);\n        }\n        return { relativePath, absolutePath };\n      })();\n      this.pendingFiles.set(fileName, publication);\n    }\n    try {\n      const location = await publication;\n      this.fileLocations.set(image.id, location);\n      return location;\n    } finally {\n      this.pendingFiles.delete(fileName);\n    }\n  }\n\n  /** Resolve any stored report image reference to a data URI. */\n  loadDataUri(refInput: unknown): string {\n    const ref = normalizeStoredImageRef(refInput);\n    if (!ref) {\n      throw new Error('ReportImageStore: invalid image reference');\n    }\n\n    const resolved = resolveImageSource(ref, {\n      reportPath: this.reportPath,\n      imageDirectory: this.screenshotsDir,\n    });\n\n    if (resolved.type === 'data-uri') {\n      return resolved.dataUri;\n    }\n\n    const data = readFileSync(resolved.filePath);\n    return `data:${resolved.mimeType};base64,${data.toString('base64')}`;\n  }\n\n  /** @deprecated Use loadDataUri. */\n  loadBase64(refInput: unknown): string {\n    const screenshotRef = normalizeScreenshotRef(refInput);\n    if (!screenshotRef) {\n      throw new Error('ReportImageStore: invalid screenshot reference');\n    }\n    return this.loadDataUri(screenshotRef);\n  }\n\n  cleanup(): void {\n    if (\n      this.mode === 'directory' &&\n      this.screenshotsDir &&\n      existsSync(this.screenshotsDir)\n    ) {\n      rmSync(this.screenshotsDir, { recursive: true, force: true });\n    }\n    this.writtenInlineIds.clear();\n    this.writtenFileIds.clear();\n    this.fileLocations.clear();\n    this.pendingFiles.clear();\n    this.existingInlineImageIdsPromise = undefined;\n  }\n}\n\n/** @deprecated Use ReportImageStore. */\nexport { ReportImageStore as ScreenshotStore };\n"],"names":["base64ImageDataUrlPattern","parseBase64ImageDataUrl","imageUrl","match","Error","mimeType","imageFileExtensionForMimeType","isBase64ImageDataUrl","value","reportImageFileName","extension","bytes","hash","createHash","resolveImageSource","refInput","options","ref","normalizeStoredImageRef","id","resolveReportRelativePath","filePath","isAbsolute","join","dirname","explicitFilePath","existsSync","inlineDataUri","extractImageByIdSync","imageRefFileExtension","companionImagePath","siblingImagePath","resolveScreenshotSource","normalizeScreenshotRef","ReportImageStore","file","screenshot","shouldWriteFileCopy","fileRef","rawBase64","Buffer","fileLocation","imageIds","collectImageScriptIds","imageId","error","relativePath","absolutePath","image","screenshotsDir","mkdirSync","known","fileName","publication","writeReportArtifactFile","location","resolved","data","readFileSync","screenshotRef","rmSync","undefined","Set","Map","Boolean"],"mappings":";;;;;;;;;;;;;;;;AA6BA,MAAMA,4BACJ;AAgBK,SAASC,wBAAwBC,QAAgB;IAKtD,MAAMC,QAAQD,SAAS,KAAK,CAACF;IAC7B,IAAI,CAACG,OACH,MAAM,IAAIC,MACR;IAGJ,MAAMC,WAAWF,KAAK,CAAC,EAAE,CAAC,WAAW;IACrC,OAAO;QACLE;QACA,WAAWC,8BAA8BD;QACzC,WAAWF,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;IACrC;AACF;AAEO,SAASI,qBAAqBC,KAAa;IAChD,OAAOR,0BAA0B,IAAI,CAACQ;AACxC;AAGO,SAASC,oBACdC,SAAiB,EACjBC,KAAiB;IAEjB,MAAMC,OAAOC,WAAW,UACrB,MAAM,CAACH,WACP,MAAM,CAAC,MACP,MAAM,CAACC,OACP,MAAM,CAAC;IACV,OAAO,GAAGC,KAAK,CAAC,EAAEF,WAAW;AAC/B;AAEO,SAASI,mBACdC,QAAiB,EACjBC,OAKC;IAED,MAAMC,MAAMC,wBAAwBH;IACpC,MAAMI,KAAKF,KAAK,MAAMD,QAAQ,UAAU;IACxC,MAAMX,WAAWY,KAAK,YAAYD,QAAQ,gBAAgB;IAE1D,IAAI,CAACG,MAAM,CAACd,UACV,MAAM,IAAID,MACR;IAIJ,MAAMgB,4BAA4B,CAACC,WACjCC,WAAWD,YACPA,WACAE,KAAKC,QAAQR,QAAQ,UAAU,GAAGK;IAExC,IAAIJ,KAAK,YAAY,QAAQ;QAC3B,IAAI,CAACA,IAAI,IAAI,EACX,MAAM,IAAIb,MACR,CAAC,6BAA6B,EAAEa,IAAI,EAAE,CAAC,mBAAmB,CAAC;QAI/D,MAAMQ,mBAAmBL,0BAA0BH,IAAI,IAAI;QAC3D,IAAIS,WAAWD,mBACb,OAAO;YACL,MAAM;YACNN;YACAd;YACA,UAAUoB;QACZ;IAEJ;IAEA,MAAME,gBAAgBC,qBAAqBZ,QAAQ,UAAU,EAAEG;IAC/D,IAAIQ,eACF,OAAO;QACL,MAAM;QACNR;QACAd;QACA,SAASsB;IACX;IAGF,MAAMjB,YAAYO,MACdY,sBAAsBZ,OACtBX,8BAA8BU,QAAQ,gBAAgB;IAC1D,IAAIA,QAAQ,cAAc,EAAE;QAC1B,MAAMc,qBAAqBP,KACzBP,QAAQ,cAAc,EACtB,GAAGG,GAAG,CAAC,EAAET,WAAW;QAEtB,IAAIgB,WAAWI,qBACb,OAAO;YACL,MAAM;YACNX;YACAd;YACA,UAAUyB;QACZ;IAEJ;IACA,MAAMC,mBAAmBR,KACvBC,QAAQR,QAAQ,UAAU,GAC1B,eACA,GAAGG,GAAG,CAAC,EAAET,WAAW;IAEtB,IAAIgB,WAAWK,mBACb,OAAO;QACL,MAAM;QACNZ;QACAd;QACA,UAAU0B;IACZ;IAGF,MAAM,IAAI3B,MACR,CAAC,wCAAwC,EAAEe,GAAG,OAAO,EAAEH,QAAQ,UAAU,EAAE;AAE/E;AAEO,SAASgB,wBACdjB,QAAiB,EACjBC,OAKC;IAED,MAAMC,MAAMgB,uBAAuBlB;IACnC,OAAOD,mBAAmBG,KAAKD;AACjC;AAGO,MAAMkB;IA4CX,mBAAyB;QACvB,IAAI,CAAC,iBAAiB,GAAG;IAC3B;IAGA,YAAYf,EAAU,EAAU;QAC9B,MAAMgB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAChB;QACpC,IAAI,CAACgB,MACH,MAAM,IAAI/B,MAAM,CAAC,wBAAwB,EAAEe,GAAG,sBAAsB,CAAC;QACvE,OAAOgB,KAAK,YAAY;IAC1B;IAEA,MAAM,QAAQC,UAA0B,EAA0B;QAChE,MAAM,IAAI,CAAC,kCAAkC;QAC7C,MAAMC,sBACJ,AAAc,gBAAd,IAAI,CAAC,IAAI,IAAoB,IAAI,CAAC,iBAAiB;QACrD,MAAMC,UAAUD,sBACZ,MAAM,IAAI,CAAC,2BAA2B,CAACD,YAAY;YACjD,iBAAiB,AAAc,gBAAd,IAAI,CAAC,IAAI;QAC5B,KACA;QAEJ,IAAI,AAAc,aAAd,IAAI,CAAC,IAAI,EAAe;YAC1B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EACxB,MAAM,IAAIhC,MACR;YAGJ,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAACgC,WAAW,EAAE,GAAG;gBAC7C,MAAM,IAAI,CAAC,gBAAgB,CAACA,WAAW,EAAE,EAAEA,WAAW,MAAM;gBAC5D,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAACA,WAAW,EAAE;YACzC;YACA,OAAOA,WAAW,mBAAmB,CAAC,IAAI,CAAC,UAAU;QACvD;QAEA,IAAI,CAACE,SACH,MAAM,IAAIlC,MACR;QAGJ,OAAOkC;IACT;IAGA,MAAM,sBAAsBpC,QAAgB,EAAwB;QAClE,MAAM,IAAI,CAAC,kCAAkC;QAC7C,MAAM,EAAEG,QAAQ,EAAEK,SAAS,EAAE6B,SAAS,EAAE,GACtCtC,wBAAwBC;QAC1B,MAAMiB,KAAK,CAAC,UAAU,EAAEN,WAAW,UAChC,MAAM,CAACR,UACP,MAAM,CAAC,MACP,MAAM,CAACmC,OAAO,IAAI,CAACD,WAAW,WAC9B,MAAM,CAAC,QAAQ;QAClB,MAAMF,sBACJ,AAAc,gBAAd,IAAI,CAAC,IAAI,IAAoB,IAAI,CAAC,iBAAiB;QACrD,MAAMI,eAAeJ,sBACjB,MAAM,IAAI,CAAC,sBAAsB,CAAC;YAAElB;YAAIT;YAAW6B;QAAU,KAC7D;QAEJ,IAAItB;QACJ,IAAI,AAAc,aAAd,IAAI,CAAC,IAAI,EAAe;YAC1B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EACxB,MAAM,IAAIb,MACR;YAGJ,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAACe,KAAK;gBAClC,MAAM,IAAI,CAAC,gBAAgB,CAACA,IAAIjB;gBAChC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAACiB;YAC5B;YACAF,MAAM;gBACJ,MAAM;gBACNE;gBACAd;gBACA,SAAS;YACX;QACF,OAAO;YACL,IAAI,CAACoC,cACH,MAAM,IAAIrC,MACR;YAGJa,MAAM;gBACJ,MAAM;gBACNE;gBACAd;gBACA,SAAS;gBACT,MAAMoC,aAAa,YAAY;YACjC;QACF;QACA,OAAOxB;IACT;IAEA,MAAc,qCAAoD;QAChE,IAAI,CAAC,IAAI,CAAC,+BAA+B,EAAE;QAC3C,IAAI,CAAC,IAAI,CAAC,6BAA6B,EACrC,IAAI,CAAC,6BAA6B,GAAI;YACpC,IAAI;gBACF,MAAMyB,WAAW,MAAMC,sBAAsB,IAAI,CAAC,UAAU;gBAC5D,KAAK,MAAMC,WAAWF,SAAU,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAACE;YAC5D,EAAE,OAAOC,OAAO;gBACd,IACE,AAAiB,YAAjB,OAAOA,SACPA,AAAU,SAAVA,SACA,UAAUA,SACVA,AAAe,aAAfA,MAAM,IAAI,EAEV;gBAEF,MAAM,IAAIzC,MACR,CAAC,8DAA8D,EAAE,IAAI,CAAC,UAAU,EAAE,EAClF;oBAAE,OAAOyC;gBAAM;YAEnB;QACF;QAEF,MAAM,IAAI,CAAC,6BAA6B;IAC1C;IAEA,MAAc,4BACZT,UAA0B,EAC1BpB,OAEC,EACuB;QACxB,MAAM,EAAE8B,YAAY,EAAEC,YAAY,EAAE,GAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAACX,WAAW,EAAE,KACnC,MAAM,IAAI,CAAC,sBAAsB,CAAC;YACjC,IAAIA,WAAW,EAAE;YACjB,WAAWA,WAAW,SAAS;YAC/B,WAAWA,WAAW,SAAS;QACjC;QAEF,IAAIpB,QAAQ,eAAe,EACzB,OAAOoB,WAAW,mBAAmB,CAACU,cAAcC;QAGtD,OAAOX,WAAW,yBAAyB,CAACU,cAAcC;IAC5D;IAEA,MAAc,uBAAuBC,KAIpC,EAA2D;QAC1D,MAAMC,iBAAiB,IAAI,CAAC,cAAc;QAC1C,IAAI,CAACA,gBACH,MAAM,IAAI7C,MACR;QAGJ,IAAI,CAACsB,WAAWuB,iBACdC,UAAUD,gBAAgB;YAAE,WAAW;QAAK;QAG9C,MAAME,QAAQ,IAAI,CAAC,aAAa,CAAC,GAAG,CAACH,MAAM,EAAE;QAC7C,IAAIG,OAAO,OAAOA;QAClB,MAAMxC,QAAQ6B,OAAO,IAAI,CAACQ,MAAM,SAAS,EAAE;QAC3C,MAAMI,WAAW3C,oBAAoBuC,MAAM,SAAS,EAAErC;QACtD,MAAMmC,eAAe,CAAC,cAAc,EAAEM,UAAU;QAChD,MAAML,eAAexB,KAAK0B,gBAAgBG;QAC1C,IAAIC,cAAc,IAAI,CAAC,YAAY,CAAC,GAAG,CAACD;QACxC,IAAI,CAACC,aAAa;YAChBA,cAAe;gBACb,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAACD,WAAW;oBACtC,MAAME,wBAAwBP,cAAcpC;oBAC5C,IAAI,CAAC,cAAc,CAAC,GAAG,CAACyC;gBAC1B;gBACA,OAAO;oBAAEN;oBAAcC;gBAAa;YACtC;YACA,IAAI,CAAC,YAAY,CAAC,GAAG,CAACK,UAAUC;QAClC;QACA,IAAI;YACF,MAAME,WAAW,MAAMF;YACvB,IAAI,CAAC,aAAa,CAAC,GAAG,CAACL,MAAM,EAAE,EAAEO;YACjC,OAAOA;QACT,SAAU;YACR,IAAI,CAAC,YAAY,CAAC,MAAM,CAACH;QAC3B;IACF;IAGA,YAAYrC,QAAiB,EAAU;QACrC,MAAME,MAAMC,wBAAwBH;QACpC,IAAI,CAACE,KACH,MAAM,IAAIb,MAAM;QAGlB,MAAMoD,WAAW1C,mBAAmBG,KAAK;YACvC,YAAY,IAAI,CAAC,UAAU;YAC3B,gBAAgB,IAAI,CAAC,cAAc;QACrC;QAEA,IAAIuC,AAAkB,eAAlBA,SAAS,IAAI,EACf,OAAOA,SAAS,OAAO;QAGzB,MAAMC,OAAOC,aAAaF,SAAS,QAAQ;QAC3C,OAAO,CAAC,KAAK,EAAEA,SAAS,QAAQ,CAAC,QAAQ,EAAEC,KAAK,QAAQ,CAAC,WAAW;IACtE;IAGA,WAAW1C,QAAiB,EAAU;QACpC,MAAM4C,gBAAgB1B,uBAAuBlB;QAC7C,IAAI,CAAC4C,eACH,MAAM,IAAIvD,MAAM;QAElB,OAAO,IAAI,CAAC,WAAW,CAACuD;IAC1B;IAEA,UAAgB;QACd,IACE,AAAc,gBAAd,IAAI,CAAC,IAAI,IACT,IAAI,CAAC,cAAc,IACnBjC,WAAW,IAAI,CAAC,cAAc,GAE9BkC,OAAO,IAAI,CAAC,cAAc,EAAE;YAAE,WAAW;YAAM,OAAO;QAAK;QAE7D,IAAI,CAAC,gBAAgB,CAAC,KAAK;QAC3B,IAAI,CAAC,cAAc,CAAC,KAAK;QACzB,IAAI,CAAC,aAAa,CAAC,KAAK;QACxB,IAAI,CAAC,YAAY,CAAC,KAAK;QACvB,IAAI,CAAC,6BAA6B,GAAGC;IACvC;IArPA,YAAY7C,OAUX,CAAE;QA/BH,uBAAiB,QAAjB;QACA,uBAAiB,cAAjB;QACA,uBAAiB,kBAAjB;QACA,uBAAiB,oBAAjB;QAIA,uBAAQ,qBAAR;QACA,uBAAiB,oBAAmB,IAAI8C;QACxC,uBAAiB,kBAAiB,IAAIA;QACtC,uBAAiB,iBAAgB,IAAIC;QAIrC,uBAAiB,gBAAe,IAAIA;QAIpC,uBAAiB,mCAAjB;QACA,uBAAQ,iCAAR;QAaE,IAAI,CAAC,IAAI,GAAG/C,QAAQ,IAAI;QACxB,IAAI,CAAC,UAAU,GAAGA,QAAQ,UAAU;QACpC,IAAI,CAAC,cAAc,GAAGA,QAAQ,cAAc;QAC5C,IAAI,CAAC,gBAAgB,GAAGA,QAAQ,gBAAgB;QAChD,IAAI,CAAC,iBAAiB,GACpBA,QAAQ,iBAAiB,IAAIA,QAAQ,cAAc,IAAI;QACzD,IAAI,CAAC,+BAA+B,GAAGgD,QACrChD,QAAQ,mBAAmB,IAAI,AAAc,aAAd,IAAI,CAAC,IAAI;IAE5C;AAkOF"}
+{"version":3,"file":"dump/screenshot-store.mjs","sources":["../../../src/dump/screenshot-store.ts"],"sourcesContent":["import { createHash } from 'node:crypto';\nimport { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';\nimport { dirname, isAbsolute, join } from 'node:path';\nimport type { ScreenshotItem } from '../screenshot-item';\nimport { collectImageScriptIds, extractImageByIdSync } from './html-utils';\nimport {\n  type ImageUrlRef,\n  type ScreenshotRef,\n  type StoredImageRef,\n  imageFileExtensionForMimeType,\n  imageRefFileExtension,\n  normalizeImageUrlRef,\n  normalizeScreenshotRef,\n  normalizeStoredImageRef,\n} from './image-reference';\nimport { writeReportArtifactFile } from './report-artifact-store';\n\nexport type {\n  ImageUrlRef,\n  ScreenshotRef,\n  StoredImageRef,\n} from './image-reference';\nexport {\n  imageRefFileExtension,\n  normalizeImageUrlRef,\n  normalizeScreenshotRef,\n  normalizeStoredImageRef,\n} from './image-reference';\n\nconst base64ImageDataUrlPattern =\n  /^data:(image\\/[a-z0-9.+-]+)(?:;[^,]*)?;base64,([a-z0-9+/=\\s]+)$/i;\ntype ResolvedImageSource =\n  | {\n      type: 'data-uri';\n      id: string;\n      mimeType: StoredImageRef['mimeType'];\n      dataUri: string;\n    }\n  | {\n      type: 'file';\n      id: string;\n      mimeType: StoredImageRef['mimeType'];\n      filePath: string;\n    };\n\n/** Parse and normalize a base64 image data URL for report persistence. */\nexport function parseBase64ImageDataUrl(imageUrl: string): {\n  mimeType: `image/${string}`;\n  extension: string;\n  rawBase64: string;\n} {\n  const match = imageUrl.match(base64ImageDataUrlPattern);\n  if (!match) {\n    throw new Error(\n      'ReportImageStore: reference image must be a valid base64 image data URL',\n    );\n  }\n  const mimeType = match[1].toLowerCase() as `image/${string}`;\n  return {\n    mimeType,\n    extension: imageFileExtensionForMimeType(mimeType),\n    rawBase64: match[2].replace(/\\s/g, ''),\n  };\n}\n\nexport function isBase64ImageDataUrl(value: string): boolean {\n  return base64ImageDataUrlPattern.test(value);\n}\n\n/** Stable asset names allow different observations to share the same bytes. */\nexport function reportImageFileName(\n  extension: string,\n  bytes: Uint8Array,\n): string {\n  const hash = createHash('sha256')\n    .update(extension)\n    .update('\\0')\n    .update(bytes)\n    .digest('hex');\n  return `${hash}.${extension}`;\n}\n\nexport function resolveImageSource(\n  refInput: unknown,\n  options: {\n    reportPath: string;\n    imageDirectory?: string;\n    fallbackId?: string;\n    fallbackMimeType?: ScreenshotRef['mimeType'];\n  },\n): ResolvedImageSource {\n  const ref = normalizeStoredImageRef(refInput);\n  const id = ref?.id ?? options.fallbackId;\n  const mimeType = ref?.mimeType ?? options.fallbackMimeType;\n\n  if (!id || !mimeType) {\n    throw new Error(\n      'ReportImageStore: image id and mimeType are required to resolve an image',\n    );\n  }\n\n  const resolveReportRelativePath = (filePath: string): string =>\n    isAbsolute(filePath)\n      ? filePath\n      : join(dirname(options.reportPath), filePath);\n\n  if (ref?.storage === 'file') {\n    if (!ref.path) {\n      throw new Error(\n        `ReportImageStore: image ref \"${ref.id}\" missing file path`,\n      );\n    }\n\n    const explicitFilePath = resolveReportRelativePath(ref.path);\n    if (existsSync(explicitFilePath)) {\n      return {\n        type: 'file',\n        id,\n        mimeType,\n        filePath: explicitFilePath,\n      };\n    }\n  }\n\n  const inlineDataUri = extractImageByIdSync(options.reportPath, id);\n  if (inlineDataUri) {\n    return {\n      type: 'data-uri',\n      id,\n      mimeType,\n      dataUri: inlineDataUri,\n    };\n  }\n\n  const extension = ref\n    ? imageRefFileExtension(ref)\n    : imageFileExtensionForMimeType(options.fallbackMimeType!);\n  if (options.imageDirectory) {\n    const companionImagePath = join(\n      options.imageDirectory,\n      `${id}.${extension}`,\n    );\n    if (existsSync(companionImagePath)) {\n      return {\n        type: 'file',\n        id,\n        mimeType,\n        filePath: companionImagePath,\n      };\n    }\n  }\n  const siblingImagePath = join(\n    dirname(options.reportPath),\n    'screenshots',\n    `${id}.${extension}`,\n  );\n  if (existsSync(siblingImagePath)) {\n    return {\n      type: 'file',\n      id,\n      mimeType,\n      filePath: siblingImagePath,\n    };\n  }\n\n  throw new Error(\n    `ReportImageStore: cannot resolve image \"${id}\" from ${options.reportPath}`,\n  );\n}\n\nexport function resolveScreenshotSource(\n  refInput: unknown,\n  options: {\n    reportPath: string;\n    imageDirectory?: string;\n    fallbackId?: string;\n    fallbackMimeType?: ScreenshotRef['mimeType'];\n  },\n): ResolvedImageSource {\n  const ref = normalizeScreenshotRef(refInput);\n  return resolveImageSource(ref, options);\n}\n\n/** Persists and restores every image asset referenced by a report. */\nexport class ReportImageStore {\n  private readonly mode: 'inline' | 'directory';\n  private readonly reportPath: string;\n  private readonly screenshotsDir?: string;\n  private readonly writeInlineImage?: (\n    id: string,\n    base64: string,\n  ) => void | Promise<void>;\n  private alsoWriteFileCopy: boolean;\n  private readonly writtenInlineIds = new Set<string>();\n  private readonly writtenFileIds = new Set<string>();\n  private readonly fileLocations = new Map<\n    string,\n    { relativePath: string; absolutePath: string }\n  >();\n  private readonly pendingFiles = new Map<\n    string,\n    Promise<{ relativePath: string; absolutePath: string }>\n  >();\n  private readonly shouldReuseExistingInlineReport: boolean;\n  private existingInlineImageIdsPromise?: Promise<void>;\n\n  constructor(options: {\n    mode: 'inline' | 'directory';\n    reportPath: string;\n    screenshotsDir?: string;\n    writeInlineImage?: (id: string, base64: string) => void | Promise<void>;\n    alsoWriteFileCopy?: boolean;\n    /** @deprecated Use alsoWriteFileCopy instead. */\n    ensureFileCopy?: boolean;\n    /** Reuse inline image assets already present in reportPath. */\n    reuseExistingReport?: boolean;\n  }) {\n    this.mode = options.mode;\n    this.reportPath = options.reportPath;\n    this.screenshotsDir = options.screenshotsDir;\n    this.writeInlineImage = options.writeInlineImage;\n    this.alsoWriteFileCopy =\n      options.alsoWriteFileCopy ?? options.ensureFileCopy ?? false;\n    this.shouldReuseExistingInlineReport = Boolean(\n      options.reuseExistingReport && this.mode === 'inline',\n    );\n  }\n\n  enableFileCopies(): void {\n    this.alsoWriteFileCopy = true;\n  }\n\n  /** Resolve an observation/reference ID to its content-addressed asset. */\n  getFilePath(id: string): string {\n    const file = this.fileLocations.get(id);\n    if (!file)\n      throw new Error(`ReportImageStore: image ${id} has no persisted file`);\n    return file.relativePath;\n  }\n\n  /** Keep diagnostic bytes without changing the screenshot's report location. */\n  persistFileCopy(screenshot: ScreenshotItem): Promise<ScreenshotRef> {\n    return this.persistToSharedFileIfNeeded(screenshot, {\n      markAsPersisted: false,\n    });\n  }\n\n  async persist(screenshot: ScreenshotItem): Promise<ScreenshotRef> {\n    await this.ensureExistingInlineImageIdsLoaded();\n    const shouldWriteFileCopy =\n      this.mode === 'directory' || this.alsoWriteFileCopy;\n    const fileRef = shouldWriteFileCopy\n      ? await this.persistToSharedFileIfNeeded(screenshot, {\n          markAsPersisted: this.mode === 'directory',\n        })\n      : null;\n\n    if (this.mode === 'inline') {\n      if (!this.writeInlineImage) {\n        throw new Error(\n          'ReportImageStore: writeInlineImage is required in inline mode',\n        );\n      }\n      if (!this.writtenInlineIds.has(screenshot.id)) {\n        await this.writeInlineImage(screenshot.id, screenshot.base64);\n        this.writtenInlineIds.add(screenshot.id);\n      }\n      return screenshot.markPersistedInline(this.reportPath);\n    }\n\n    if (!fileRef) {\n      throw new Error(\n        'ReportImageStore: file persistence is required in directory mode',\n      );\n    }\n    return fileRef;\n  }\n\n  /** Persist a base64 multimodal prompt image and return its content-addressed ref. */\n  async persistReferenceImage(imageUrl: string): Promise<ImageUrlRef> {\n    await this.ensureExistingInlineImageIdsLoaded();\n    const { mimeType, extension, rawBase64 } =\n      parseBase64ImageDataUrl(imageUrl);\n    const id = `reference-${createHash('sha256')\n      .update(mimeType)\n      .update('\\0')\n      .update(Buffer.from(rawBase64, 'base64'))\n      .digest('hex')}`;\n    const shouldWriteFileCopy =\n      this.mode === 'directory' || this.alsoWriteFileCopy;\n    const fileLocation = shouldWriteFileCopy\n      ? await this.writeImageFileIfNeeded({ id, extension, rawBase64 })\n      : null;\n\n    let ref: ImageUrlRef;\n    if (this.mode === 'inline') {\n      if (!this.writeInlineImage) {\n        throw new Error(\n          'ReportImageStore: writeInlineImage is required in inline mode',\n        );\n      }\n      if (!this.writtenInlineIds.has(id)) {\n        await this.writeInlineImage(id, imageUrl);\n        this.writtenInlineIds.add(id);\n      }\n      ref = {\n        type: 'midscene_image_url_ref',\n        id,\n        mimeType,\n        storage: 'inline',\n      };\n    } else {\n      if (!fileLocation) {\n        throw new Error(\n          'ReportImageStore: file persistence is required in directory mode',\n        );\n      }\n      ref = {\n        type: 'midscene_image_url_ref',\n        id,\n        mimeType,\n        storage: 'file',\n        path: fileLocation.relativePath,\n      };\n    }\n    return ref;\n  }\n\n  private async ensureExistingInlineImageIdsLoaded(): Promise<void> {\n    if (!this.shouldReuseExistingInlineReport) return;\n    if (!this.existingInlineImageIdsPromise) {\n      this.existingInlineImageIdsPromise = (async () => {\n        try {\n          const imageIds = await collectImageScriptIds(this.reportPath);\n          for (const imageId of imageIds) this.writtenInlineIds.add(imageId);\n        } catch (error) {\n          if (\n            typeof error === 'object' &&\n            error !== null &&\n            'code' in error &&\n            error.code === 'ENOENT'\n          ) {\n            return;\n          }\n          throw new Error(\n            `ReportImageStore: failed to index existing inline images from ${this.reportPath}`,\n            { cause: error },\n          );\n        }\n      })();\n    }\n    await this.existingInlineImageIdsPromise;\n  }\n\n  private async persistToSharedFileIfNeeded(\n    screenshot: ScreenshotItem,\n    options: {\n      markAsPersisted: boolean;\n    },\n  ): Promise<ScreenshotRef> {\n    const { relativePath, absolutePath } =\n      this.fileLocations.get(screenshot.id) ??\n      (await this.writeImageFileIfNeeded({\n        id: screenshot.id,\n        extension: screenshot.extension,\n        rawBase64: screenshot.rawBase64,\n      }));\n\n    if (options.markAsPersisted) {\n      return screenshot.markPersistedToPath(relativePath, absolutePath);\n    }\n\n    return screenshot.registerPersistedFileCopy(relativePath, absolutePath);\n  }\n\n  private async writeImageFileIfNeeded(image: {\n    id: string;\n    extension: string;\n    rawBase64: string;\n  }): Promise<{ relativePath: string; absolutePath: string }> {\n    const screenshotsDir = this.screenshotsDir;\n    if (!screenshotsDir) {\n      throw new Error(\n        'ReportImageStore: screenshotsDir is required when file persistence is enabled',\n      );\n    }\n    if (!existsSync(screenshotsDir)) {\n      mkdirSync(screenshotsDir, { recursive: true });\n    }\n\n    const known = this.fileLocations.get(image.id);\n    if (known) return known;\n    const bytes = Buffer.from(image.rawBase64, 'base64');\n    const fileName = reportImageFileName(image.extension, bytes);\n    const relativePath = `./screenshots/${fileName}`;\n    const absolutePath = join(screenshotsDir, fileName);\n    let publication = this.pendingFiles.get(fileName);\n    if (!publication) {\n      publication = (async () => {\n        if (!this.writtenFileIds.has(fileName)) {\n          await writeReportArtifactFile(absolutePath, bytes);\n          this.writtenFileIds.add(fileName);\n        }\n        return { relativePath, absolutePath };\n      })();\n      this.pendingFiles.set(fileName, publication);\n    }\n    try {\n      const location = await publication;\n      this.fileLocations.set(image.id, location);\n      return location;\n    } finally {\n      this.pendingFiles.delete(fileName);\n    }\n  }\n\n  /** Resolve any stored report image reference to a data URI. */\n  loadDataUri(refInput: unknown): string {\n    const ref = normalizeStoredImageRef(refInput);\n    if (!ref) {\n      throw new Error('ReportImageStore: invalid image reference');\n    }\n\n    const resolved = resolveImageSource(ref, {\n      reportPath: this.reportPath,\n      imageDirectory: this.screenshotsDir,\n    });\n\n    if (resolved.type === 'data-uri') {\n      return resolved.dataUri;\n    }\n\n    const data = readFileSync(resolved.filePath);\n    return `data:${resolved.mimeType};base64,${data.toString('base64')}`;\n  }\n\n  /** @deprecated Use loadDataUri. */\n  loadBase64(refInput: unknown): string {\n    const screenshotRef = normalizeScreenshotRef(refInput);\n    if (!screenshotRef) {\n      throw new Error('ReportImageStore: invalid screenshot reference');\n    }\n    return this.loadDataUri(screenshotRef);\n  }\n\n  cleanup(): void {\n    if (\n      this.mode === 'directory' &&\n      this.screenshotsDir &&\n      existsSync(this.screenshotsDir)\n    ) {\n      rmSync(this.screenshotsDir, { recursive: true, force: true });\n    }\n    this.writtenInlineIds.clear();\n    this.writtenFileIds.clear();\n    this.fileLocations.clear();\n    this.pendingFiles.clear();\n    this.existingInlineImageIdsPromise = undefined;\n  }\n}\n\n/** @deprecated Use ReportImageStore. */\nexport { ReportImageStore as ScreenshotStore };\n"],"names":["base64ImageDataUrlPattern","parseBase64ImageDataUrl","imageUrl","match","Error","mimeType","imageFileExtensionForMimeType","isBase64ImageDataUrl","value","reportImageFileName","extension","bytes","hash","createHash","resolveImageSource","refInput","options","ref","normalizeStoredImageRef","id","resolveReportRelativePath","filePath","isAbsolute","join","dirname","explicitFilePath","existsSync","inlineDataUri","extractImageByIdSync","imageRefFileExtension","companionImagePath","siblingImagePath","resolveScreenshotSource","normalizeScreenshotRef","ReportImageStore","file","screenshot","shouldWriteFileCopy","fileRef","rawBase64","Buffer","fileLocation","imageIds","collectImageScriptIds","imageId","error","relativePath","absolutePath","image","screenshotsDir","mkdirSync","known","fileName","publication","writeReportArtifactFile","location","resolved","data","readFileSync","screenshotRef","rmSync","undefined","Set","Map","Boolean"],"mappings":";;;;;;;;;;;;;;;;AA6BA,MAAMA,4BACJ;AAgBK,SAASC,wBAAwBC,QAAgB;IAKtD,MAAMC,QAAQD,SAAS,KAAK,CAACF;IAC7B,IAAI,CAACG,OACH,MAAM,IAAIC,MACR;IAGJ,MAAMC,WAAWF,KAAK,CAAC,EAAE,CAAC,WAAW;IACrC,OAAO;QACLE;QACA,WAAWC,8BAA8BD;QACzC,WAAWF,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;IACrC;AACF;AAEO,SAASI,qBAAqBC,KAAa;IAChD,OAAOR,0BAA0B,IAAI,CAACQ;AACxC;AAGO,SAASC,oBACdC,SAAiB,EACjBC,KAAiB;IAEjB,MAAMC,OAAOC,WAAW,UACrB,MAAM,CAACH,WACP,MAAM,CAAC,MACP,MAAM,CAACC,OACP,MAAM,CAAC;IACV,OAAO,GAAGC,KAAK,CAAC,EAAEF,WAAW;AAC/B;AAEO,SAASI,mBACdC,QAAiB,EACjBC,OAKC;IAED,MAAMC,MAAMC,wBAAwBH;IACpC,MAAMI,KAAKF,KAAK,MAAMD,QAAQ,UAAU;IACxC,MAAMX,WAAWY,KAAK,YAAYD,QAAQ,gBAAgB;IAE1D,IAAI,CAACG,MAAM,CAACd,UACV,MAAM,IAAID,MACR;IAIJ,MAAMgB,4BAA4B,CAACC,WACjCC,WAAWD,YACPA,WACAE,KAAKC,QAAQR,QAAQ,UAAU,GAAGK;IAExC,IAAIJ,KAAK,YAAY,QAAQ;QAC3B,IAAI,CAACA,IAAI,IAAI,EACX,MAAM,IAAIb,MACR,CAAC,6BAA6B,EAAEa,IAAI,EAAE,CAAC,mBAAmB,CAAC;QAI/D,MAAMQ,mBAAmBL,0BAA0BH,IAAI,IAAI;QAC3D,IAAIS,WAAWD,mBACb,OAAO;YACL,MAAM;YACNN;YACAd;YACA,UAAUoB;QACZ;IAEJ;IAEA,MAAME,gBAAgBC,qBAAqBZ,QAAQ,UAAU,EAAEG;IAC/D,IAAIQ,eACF,OAAO;QACL,MAAM;QACNR;QACAd;QACA,SAASsB;IACX;IAGF,MAAMjB,YAAYO,MACdY,sBAAsBZ,OACtBX,8BAA8BU,QAAQ,gBAAgB;IAC1D,IAAIA,QAAQ,cAAc,EAAE;QAC1B,MAAMc,qBAAqBP,KACzBP,QAAQ,cAAc,EACtB,GAAGG,GAAG,CAAC,EAAET,WAAW;QAEtB,IAAIgB,WAAWI,qBACb,OAAO;YACL,MAAM;YACNX;YACAd;YACA,UAAUyB;QACZ;IAEJ;IACA,MAAMC,mBAAmBR,KACvBC,QAAQR,QAAQ,UAAU,GAC1B,eACA,GAAGG,GAAG,CAAC,EAAET,WAAW;IAEtB,IAAIgB,WAAWK,mBACb,OAAO;QACL,MAAM;QACNZ;QACAd;QACA,UAAU0B;IACZ;IAGF,MAAM,IAAI3B,MACR,CAAC,wCAAwC,EAAEe,GAAG,OAAO,EAAEH,QAAQ,UAAU,EAAE;AAE/E;AAEO,SAASgB,wBACdjB,QAAiB,EACjBC,OAKC;IAED,MAAMC,MAAMgB,uBAAuBlB;IACnC,OAAOD,mBAAmBG,KAAKD;AACjC;AAGO,MAAMkB;IA4CX,mBAAyB;QACvB,IAAI,CAAC,iBAAiB,GAAG;IAC3B;IAGA,YAAYf,EAAU,EAAU;QAC9B,MAAMgB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAChB;QACpC,IAAI,CAACgB,MACH,MAAM,IAAI/B,MAAM,CAAC,wBAAwB,EAAEe,GAAG,sBAAsB,CAAC;QACvE,OAAOgB,KAAK,YAAY;IAC1B;IAGA,gBAAgBC,UAA0B,EAA0B;QAClE,OAAO,IAAI,CAAC,2BAA2B,CAACA,YAAY;YAClD,iBAAiB;QACnB;IACF;IAEA,MAAM,QAAQA,UAA0B,EAA0B;QAChE,MAAM,IAAI,CAAC,kCAAkC;QAC7C,MAAMC,sBACJ,AAAc,gBAAd,IAAI,CAAC,IAAI,IAAoB,IAAI,CAAC,iBAAiB;QACrD,MAAMC,UAAUD,sBACZ,MAAM,IAAI,CAAC,2BAA2B,CAACD,YAAY;YACjD,iBAAiB,AAAc,gBAAd,IAAI,CAAC,IAAI;QAC5B,KACA;QAEJ,IAAI,AAAc,aAAd,IAAI,CAAC,IAAI,EAAe;YAC1B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EACxB,MAAM,IAAIhC,MACR;YAGJ,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAACgC,WAAW,EAAE,GAAG;gBAC7C,MAAM,IAAI,CAAC,gBAAgB,CAACA,WAAW,EAAE,EAAEA,WAAW,MAAM;gBAC5D,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAACA,WAAW,EAAE;YACzC;YACA,OAAOA,WAAW,mBAAmB,CAAC,IAAI,CAAC,UAAU;QACvD;QAEA,IAAI,CAACE,SACH,MAAM,IAAIlC,MACR;QAGJ,OAAOkC;IACT;IAGA,MAAM,sBAAsBpC,QAAgB,EAAwB;QAClE,MAAM,IAAI,CAAC,kCAAkC;QAC7C,MAAM,EAAEG,QAAQ,EAAEK,SAAS,EAAE6B,SAAS,EAAE,GACtCtC,wBAAwBC;QAC1B,MAAMiB,KAAK,CAAC,UAAU,EAAEN,WAAW,UAChC,MAAM,CAACR,UACP,MAAM,CAAC,MACP,MAAM,CAACmC,OAAO,IAAI,CAACD,WAAW,WAC9B,MAAM,CAAC,QAAQ;QAClB,MAAMF,sBACJ,AAAc,gBAAd,IAAI,CAAC,IAAI,IAAoB,IAAI,CAAC,iBAAiB;QACrD,MAAMI,eAAeJ,sBACjB,MAAM,IAAI,CAAC,sBAAsB,CAAC;YAAElB;YAAIT;YAAW6B;QAAU,KAC7D;QAEJ,IAAItB;QACJ,IAAI,AAAc,aAAd,IAAI,CAAC,IAAI,EAAe;YAC1B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EACxB,MAAM,IAAIb,MACR;YAGJ,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAACe,KAAK;gBAClC,MAAM,IAAI,CAAC,gBAAgB,CAACA,IAAIjB;gBAChC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAACiB;YAC5B;YACAF,MAAM;gBACJ,MAAM;gBACNE;gBACAd;gBACA,SAAS;YACX;QACF,OAAO;YACL,IAAI,CAACoC,cACH,MAAM,IAAIrC,MACR;YAGJa,MAAM;gBACJ,MAAM;gBACNE;gBACAd;gBACA,SAAS;gBACT,MAAMoC,aAAa,YAAY;YACjC;QACF;QACA,OAAOxB;IACT;IAEA,MAAc,qCAAoD;QAChE,IAAI,CAAC,IAAI,CAAC,+BAA+B,EAAE;QAC3C,IAAI,CAAC,IAAI,CAAC,6BAA6B,EACrC,IAAI,CAAC,6BAA6B,GAAI;YACpC,IAAI;gBACF,MAAMyB,WAAW,MAAMC,sBAAsB,IAAI,CAAC,UAAU;gBAC5D,KAAK,MAAMC,WAAWF,SAAU,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAACE;YAC5D,EAAE,OAAOC,OAAO;gBACd,IACE,AAAiB,YAAjB,OAAOA,SACPA,AAAU,SAAVA,SACA,UAAUA,SACVA,AAAe,aAAfA,MAAM,IAAI,EAEV;gBAEF,MAAM,IAAIzC,MACR,CAAC,8DAA8D,EAAE,IAAI,CAAC,UAAU,EAAE,EAClF;oBAAE,OAAOyC;gBAAM;YAEnB;QACF;QAEF,MAAM,IAAI,CAAC,6BAA6B;IAC1C;IAEA,MAAc,4BACZT,UAA0B,EAC1BpB,OAEC,EACuB;QACxB,MAAM,EAAE8B,YAAY,EAAEC,YAAY,EAAE,GAClC,IAAI,CAAC,aAAa,CAAC,GAAG,CAACX,WAAW,EAAE,KACnC,MAAM,IAAI,CAAC,sBAAsB,CAAC;YACjC,IAAIA,WAAW,EAAE;YACjB,WAAWA,WAAW,SAAS;YAC/B,WAAWA,WAAW,SAAS;QACjC;QAEF,IAAIpB,QAAQ,eAAe,EACzB,OAAOoB,WAAW,mBAAmB,CAACU,cAAcC;QAGtD,OAAOX,WAAW,yBAAyB,CAACU,cAAcC;IAC5D;IAEA,MAAc,uBAAuBC,KAIpC,EAA2D;QAC1D,MAAMC,iBAAiB,IAAI,CAAC,cAAc;QAC1C,IAAI,CAACA,gBACH,MAAM,IAAI7C,MACR;QAGJ,IAAI,CAACsB,WAAWuB,iBACdC,UAAUD,gBAAgB;YAAE,WAAW;QAAK;QAG9C,MAAME,QAAQ,IAAI,CAAC,aAAa,CAAC,GAAG,CAACH,MAAM,EAAE;QAC7C,IAAIG,OAAO,OAAOA;QAClB,MAAMxC,QAAQ6B,OAAO,IAAI,CAACQ,MAAM,SAAS,EAAE;QAC3C,MAAMI,WAAW3C,oBAAoBuC,MAAM,SAAS,EAAErC;QACtD,MAAMmC,eAAe,CAAC,cAAc,EAAEM,UAAU;QAChD,MAAML,eAAexB,KAAK0B,gBAAgBG;QAC1C,IAAIC,cAAc,IAAI,CAAC,YAAY,CAAC,GAAG,CAACD;QACxC,IAAI,CAACC,aAAa;YAChBA,cAAe;gBACb,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAACD,WAAW;oBACtC,MAAME,wBAAwBP,cAAcpC;oBAC5C,IAAI,CAAC,cAAc,CAAC,GAAG,CAACyC;gBAC1B;gBACA,OAAO;oBAAEN;oBAAcC;gBAAa;YACtC;YACA,IAAI,CAAC,YAAY,CAAC,GAAG,CAACK,UAAUC;QAClC;QACA,IAAI;YACF,MAAME,WAAW,MAAMF;YACvB,IAAI,CAAC,aAAa,CAAC,GAAG,CAACL,MAAM,EAAE,EAAEO;YACjC,OAAOA;QACT,SAAU;YACR,IAAI,CAAC,YAAY,CAAC,MAAM,CAACH;QAC3B;IACF;IAGA,YAAYrC,QAAiB,EAAU;QACrC,MAAME,MAAMC,wBAAwBH;QACpC,IAAI,CAACE,KACH,MAAM,IAAIb,MAAM;QAGlB,MAAMoD,WAAW1C,mBAAmBG,KAAK;YACvC,YAAY,IAAI,CAAC,UAAU;YAC3B,gBAAgB,IAAI,CAAC,cAAc;QACrC;QAEA,IAAIuC,AAAkB,eAAlBA,SAAS,IAAI,EACf,OAAOA,SAAS,OAAO;QAGzB,MAAMC,OAAOC,aAAaF,SAAS,QAAQ;QAC3C,OAAO,CAAC,KAAK,EAAEA,SAAS,QAAQ,CAAC,QAAQ,EAAEC,KAAK,QAAQ,CAAC,WAAW;IACtE;IAGA,WAAW1C,QAAiB,EAAU;QACpC,MAAM4C,gBAAgB1B,uBAAuBlB;QAC7C,IAAI,CAAC4C,eACH,MAAM,IAAIvD,MAAM;QAElB,OAAO,IAAI,CAAC,WAAW,CAACuD;IAC1B;IAEA,UAAgB;QACd,IACE,AAAc,gBAAd,IAAI,CAAC,IAAI,IACT,IAAI,CAAC,cAAc,IACnBjC,WAAW,IAAI,CAAC,cAAc,GAE9BkC,OAAO,IAAI,CAAC,cAAc,EAAE;YAAE,WAAW;YAAM,OAAO;QAAK;QAE7D,IAAI,CAAC,gBAAgB,CAAC,KAAK;QAC3B,IAAI,CAAC,cAAc,CAAC,KAAK;QACzB,IAAI,CAAC,aAAa,CAAC,KAAK;QACxB,IAAI,CAAC,YAAY,CAAC,KAAK;QACvB,IAAI,CAAC,6BAA6B,GAAGC;IACvC;IA5PA,YAAY7C,OAUX,CAAE;QA/BH,uBAAiB,QAAjB;QACA,uBAAiB,cAAjB;QACA,uBAAiB,kBAAjB;QACA,uBAAiB,oBAAjB;QAIA,uBAAQ,qBAAR;QACA,uBAAiB,oBAAmB,IAAI8C;QACxC,uBAAiB,kBAAiB,IAAIA;QACtC,uBAAiB,iBAAgB,IAAIC;QAIrC,uBAAiB,gBAAe,IAAIA;QAIpC,uBAAiB,mCAAjB;QACA,uBAAQ,iCAAR;QAaE,IAAI,CAAC,IAAI,GAAG/C,QAAQ,IAAI;QACxB,IAAI,CAAC,UAAU,GAAGA,QAAQ,UAAU;QACpC,IAAI,CAAC,cAAc,GAAGA,QAAQ,cAAc;QAC5C,IAAI,CAAC,gBAAgB,GAAGA,QAAQ,gBAAgB;QAChD,IAAI,CAAC,iBAAiB,GACpBA,QAAQ,iBAAiB,IAAIA,QAAQ,cAAc,IAAI;QACzD,IAAI,CAAC,+BAA+B,GAAGgD,QACrChD,QAAQ,mBAAmB,IAAI,AAAc,aAAd,IAAI,CAAC,IAAI;IAE5C;AAyOF"}