npm - @midscene/core - Versions diffs - 1.16.1 → 1.16.2 - Mend

@midscene/core 1.16.1 → 1.16.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (64) hide show

package/dist/es/agent/utils.mjs CHANGED Viewed

@@ -4,8 +4,8 @@ import { ScreenshotItem } from "../screenshot-item.mjs";
 import { uploadTestInfoToServer } from "../utils.mjs";
 import { MIDSCENE_REPORT_QUIET, globalConfigManager } from "@midscene/shared/env";
 import { createImgBase64ByFormat, imageInfoOfBase64 } from "@midscene/shared/img";
-import { getDebug } from "@midscene/shared/logger";
-import { assert, ifInBrowser, logMsg } from "@midscene/shared/utils";
+import { getDebug, logReport } from "@midscene/shared/logger";
+import { assert, ifInBrowser } from "@midscene/shared/utils";
 import { prepareRawScreenshot } from "./screenshot-preparation.mjs";
 import { debug as external_task_cache_mjs_debug } from "./task-cache.mjs";
 import { getReportFileName } from "./report-file-name.mjs";
@@ -98,7 +98,7 @@ async function createScreenshotBoundUIContext(screenshotBase64, opt) {
 }
 function printReportMsg(filepath) {
     if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) return;
-    logMsg(`Midscene - report file updated: ${filepath}`);
+    logReport(`Midscene - report file updated: ${filepath}`);
 }
 function normalizeFilePaths(files, options = {}) {
     const { fileExists = existsSync, isInBrowser = ifInBrowser, resolvePath = resolve, wslDistroName = process.env.WSL_DISTRO_NAME, cwd = process.cwd() } = options;
@@ -153,7 +153,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.16.1";
+const getMidsceneVersion = ()=>"1.16.2";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/agent/utils.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { resolve } from 'node:path';\nimport type { PixelLocateResult } from '@/ai-model/shared/model-locate-result';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n DetailedLocateParam,\n ElementCacheFeature,\n LocateResultElement,\n Rect,\n ScrollParam,\n Size,\n UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n MIDSCENE_REPORT_QUIET,\n globalConfigManager,\n} from '@midscene/shared/env';\nimport {\n createImgBase64ByFormat,\n imageInfoOfBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, ifInBrowser, logMsg } from '@midscene/shared/utils';\nimport { prepareRawScreenshot } from './screenshot-preparation';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nexport { getReportFileName } from './report-file-name';\n\nconst agentDebug = getDebug('agent');\nconst screenshotDataUrlPattern = /^data:image\\/[a-zA-Z0-9.+-]+;base64,/i;\n\nconst inferBase64ImageFormat = (base64Body: string) => {\n if (base64Body.startsWith('iVBORw0KGgo')) {\n return 'png';\n }\n return 'jpeg';\n};\n\nconst normalizeScreenshotBase64 = (screenshotBase64: string) => {\n const trimmedBase64 = screenshotBase64.trim();\n if (screenshotDataUrlPattern.test(trimmedBase64)) {\n return trimmedBase64;\n }\n\n const base64Body = trimmedBase64.replace(/\\s/g, '');\n assert(base64Body, 'screenshotBase64 must include image data');\n return createImgBase64ByFormat(\n inferBase64ImageFormat(base64Body),\n base64Body,\n );\n};\n\nconst legacyScrollTypeMap = {\n once: 'singleAction',\n untilBottom: 'scrollToBottom',\n untilTop: 'scrollToTop',\n untilRight: 'scrollToRight',\n untilLeft: 'scrollToLeft',\n} as const;\n\nexport const normalizeScrollType = (\n scrollType: string \| undefined,\n): ScrollParam['scrollType'] \| undefined => {\n if (!scrollType) {\n return undefined;\n }\n\n if (scrollType in legacyScrollTypeMap) {\n return legacyScrollTypeMap[scrollType as keyof typeof legacyScrollTypeMap];\n }\n\n return scrollType as ScrollParam['scrollType'];\n};\n\nexport async function commonContextParser(\n interfaceInstance: AbstractInterface,\n _opt: {\n uploadServerUrl?: string;\n screenshotShrinkFactor?: number;\n },\n): Promise<UIContext> {\n const debug = getDebug('commonContextParser');\n\n assert(interfaceInstance, 'interfaceInstance is required');\n\n debug('Getting interface description');\n const description = interfaceInstance.describe?.() \|\| '';\n debug('Interface description end');\n\n debug('Uploading test info to server');\n uploadTestInfoToServer({\n testUrl: description,\n serverUrl: _opt.uploadServerUrl,\n });\n debug('UploadTestInfoToServer end');\n\n debug('will get size');\n const interfaceSize = await interfaceInstance.size();\n const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n if ((interfaceSize as unknown as { dpr: number }).dpr) {\n console.warn(\n 'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n );\n }\n\n if (!Number.isFinite(logicalWidth) \|\| !Number.isFinite(logicalHeight)) {\n throw new Error(\n `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n );\n }\n\n if (logicalWidth <= 0 \|\| logicalHeight <= 0) {\n throw new Error(\n `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n );\n }\n\n debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n const screenshotBase64 = await interfaceInstance.screenshotBase64();\n const screenshotCapturedAt = Date.now();\n assert(screenshotBase64!, 'screenshotBase64 is required');\n const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n debug('will get screenshot dimensions');\n const preparedScreenshot = await prepareRawScreenshot(screenshotBase64, {\n shrinkFactor: userShrinkFactor,\n });\n const { width: imgWidth, height: imgHeight } =\n preparedScreenshot.originalSize;\n debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n // Detect orientation mismatch between logical size and screenshot.\n // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n // size() to return portrait dimensions even when the device is landscape.\n // We detect this by comparing aspect ratios and swap if they disagree.\n const logicalIsPortrait = logicalWidth < logicalHeight;\n const screenshotIsPortrait = imgWidth < imgHeight;\n let finalLogicalWidth = logicalWidth;\n let finalLogicalHeight = logicalHeight;\n if (logicalIsPortrait !== screenshotIsPortrait) {\n debug(\n `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n );\n finalLogicalWidth = logicalHeight;\n finalLogicalHeight = logicalWidth;\n }\n\n const dpr = imgWidth / finalLogicalWidth;\n\n debug('calculated dpr:', dpr);\n\n const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n if (userShrinkFactor > 1) {\n debug(\n `Applied screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${preparedScreenshot.shotSize.width}x${preparedScreenshot.shotSize.height})`,\n );\n }\n\n return {\n shotSize: preparedScreenshot.shotSize,\n deprecatedDpr: dpr,\n screenshot: ScreenshotItem.create(\n preparedScreenshot.base64,\n screenshotCapturedAt,\n ),\n shrunkShotToLogicalRatio,\n };\n}\n\nexport async function createScreenshotBoundUIContext(\n screenshotBase64: string,\n opt: {\n screenshotSize?: Size;\n },\n): Promise<UIContext> {\n const normalizedScreenshotBase64 =\n normalizeScreenshotBase64(screenshotBase64);\n const actualScreenshotSize = await imageInfoOfBase64(\n normalizedScreenshotBase64,\n );\n if (\n opt.screenshotSize &&\n (opt.screenshotSize.width !== actualScreenshotSize.width \|\|\n opt.screenshotSize.height !== actualScreenshotSize.height)\n ) {\n agentDebug(\n 'describeElementAtPoint screenshotSize mismatch, use actual size',\n {\n provided: opt.screenshotSize,\n actual: actualScreenshotSize,\n },\n );\n }\n\n return {\n screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),\n shotSize: actualScreenshotSize,\n shrunkShotToLogicalRatio: 1,\n _isFrozen: true,\n };\n}\n\nexport function printReportMsg(filepath: string) {\n if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n return;\n }\n logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\ntype NormalizeFilePathsOptions = {\n fileExists?: (path: string) => boolean;\n isInBrowser?: boolean;\n resolvePath?: (path: string) => string;\n wslDistroName?: string;\n cwd?: string;\n};\n\nexport function normalizeFilePaths(\n files: string[],\n options: NormalizeFilePathsOptions = {},\n): string[] {\n const {\n fileExists = existsSync,\n isInBrowser = ifInBrowser,\n resolvePath = resolve,\n wslDistroName = process.env.WSL_DISTRO_NAME,\n cwd = process.cwd(),\n } = options;\n\n if (isInBrowser) {\n throw new Error('File chooser is not supported in browser environment');\n }\n\n return files.map((file) => {\n const absolutePath = resolvePath(file);\n if (!fileExists(absolutePath)) {\n throw new Error(\n `File not found: ${file}. Resolved to: ${absolutePath}. Current working directory: ${cwd}`,\n );\n }\n\n if (!wslDistroName) {\n return absolutePath;\n }\n\n const wslMount = absolutePath.match(/^\\/mnt\\/([a-z])\\//);\n if (wslMount) {\n return `${wslMount[1].toUpperCase()}:\\\\${absolutePath.slice(7).replace(/\\//g, '\\\\')}`;\n }\n\n return `\\\\\\\\wsl$\\\\${wslDistroName}${absolutePath.replace(/\\//g, '\\\\')}`;\n });\n}\n\ntype LocateParamWithMaybeLocatedPixelResult = DetailedLocateParam & {\n locatedPixelResult?: unknown;\n};\n\ntype LocateParamWithLocatedPixelResult = DetailedLocateParam & {\n locatedPixelResult: PixelLocateResult;\n};\n\nexport function ifLocateParamHasLocatedPixelResult(\n planLocateParam: LocateParamWithMaybeLocatedPixelResult,\n): planLocateParam is LocateParamWithLocatedPixelResult {\n const result = planLocateParam.locatedPixelResult;\n if (!result \|\| typeof result !== 'object' \|\| !('center' in result)) {\n return false;\n }\n const point = result.center;\n return (\n Array.isArray(point) &&\n point.length === 2 &&\n point.every((v) => typeof v === 'number' && Number.isFinite(v))\n );\n}\n\nexport function matchElementFromPlan(\n planLocateParam: LocateParamWithLocatedPixelResult,\n): LocateResultElement \| undefined {\n if (!planLocateParam) {\n return undefined;\n }\n\n const { center, rect } = planLocateParam.locatedPixelResult;\n return {\n center: [...center],\n description:\n typeof planLocateParam.prompt === 'string'\n ? planLocateParam.prompt\n : planLocateParam.prompt?.prompt \|\| '',\n ...(rect ? { rect } : {}),\n };\n}\n\nexport async function matchElementFromCache(\n context: {\n taskCache?: TaskCache;\n interfaceInstance: AbstractInterface;\n },\n cacheEntry: ElementCacheFeature \| undefined,\n cachePrompt: TUserPrompt,\n cacheable: boolean \| undefined,\n): Promise<LocateResultElement \| undefined> {\n if (!cacheEntry) {\n return undefined;\n }\n\n if (cacheable === false) {\n cacheDebug('cache disabled for prompt: %s', cachePrompt);\n return undefined;\n }\n\n if (!context.taskCache?.isCacheResultUsed) {\n return undefined;\n }\n\n if (!context.interfaceInstance.rectMatchesCacheFeature) {\n cacheDebug(\n 'interface does not implement rectMatchesCacheFeature, skip cache',\n );\n return undefined;\n }\n\n try {\n const rect =\n await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n const element: LocateResultElement = {\n center: [rect.left + rect.width / 2, rect.top + rect.height / 2],\n rect,\n description:\n typeof cachePrompt === 'string'\n ? cachePrompt\n : cachePrompt.prompt \|\| '',\n };\n\n cacheDebug('cache hit, prompt: %s', cachePrompt);\n return element;\n } catch (error) {\n cacheDebug('rectMatchesCacheFeature error: %s', error);\n return undefined;\n }\n}\n\ndeclare const __VERSION__: string \| undefined;\n\nexport const getMidsceneVersion = (): string => {\n if (typeof __VERSION__ !== 'undefined') {\n return __VERSION__;\n } else if (\n process.env.__VERSION__ &&\n process.env.__VERSION__ !== 'undefined'\n ) {\n return process.env.__VERSION__;\n }\n throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n prompt: TUserPrompt,\n): {\n textPrompt: string;\n multimodalPrompt?: TMultimodalPrompt;\n} => {\n if (typeof prompt === 'string') {\n return {\n textPrompt: prompt,\n multimodalPrompt: undefined,\n };\n }\n return {\n textPrompt: prompt.prompt,\n multimodalPrompt: prompt.images\n ? {\n images: prompt.images,\n convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n }\n : undefined,\n };\n};\n\nexport const transformLogicalElementToScreenshot = (\n element: LocateResultElement,\n shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n if (shrunkShotToLogicalRatio === 1) {\n return element;\n }\n\n return {\n ...element,\n center: [\n element.center[0] * shrunkShotToLogicalRatio,\n element.center[1] * shrunkShotToLogicalRatio,\n ],\n ...(element.rect\n ? {\n rect: transformLogicalRectToScreenshotRect(\n element.rect,\n shrunkShotToLogicalRatio,\n ),\n }\n : {}),\n };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n rect: Rect,\n shrunkShotToLogicalRatio: number,\n): Rect => {\n if (shrunkShotToLogicalRatio === 1) {\n return rect;\n }\n\n return {\n ...rect,\n left: Math.round(rect.left * shrunkShotToLogicalRatio),\n top: Math.round(rect.top * shrunkShotToLogicalRatio),\n width: Math.round(rect.width * shrunkShotToLogicalRatio),\n height: Math.round(rect.height * shrunkShotToLogicalRatio),\n };\n};\n"],"names":["agentDebug","getDebug","screenshotDataUrlPattern","inferBase64ImageFormat","base64Body","normalizeScreenshotBase64","screenshotBase64","trimmedBase64","assert","createImgBase64ByFormat","legacyScrollTypeMap","normalizeScrollType","scrollType","commonContextParser","interfaceInstance","_opt","debug","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotCapturedAt","Date","userShrinkFactor","preparedScreenshot","prepareRawScreenshot","imgWidth","imgHeight","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","dpr","shrunkShotToLogicalRatio","ScreenshotItem","createScreenshotBoundUIContext","opt","normalizedScreenshotBase64","actualScreenshotSize","imageInfoOfBase64","printReportMsg","filepath","globalConfigManager","MIDSCENE_REPORT_QUIET","logMsg","normalizeFilePaths","files","options","fileExists","existsSync","isInBrowser","ifInBrowser","resolvePath","resolve","wslDistroName","process","cwd","file","absolutePath","wslMount","ifLocateParamHasLocatedPixelResult","planLocateParam","result","point","Array","v","matchElementFromPlan","center","rect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","element","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect","Math"],"mappings":";;;;;;;;;;;AAiCA,MAAMA,aAAaC,SAAS;AAC5B,MAAMC,2BAA2B;AAEjC,MAAMC,yBAAyB,CAACC;IAC9B,IAAIA,WAAW,UAAU,CAAC,gBACxB,OAAO;IAET,OAAO;AACT;AAEA,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,gBAAgBD,iBAAiB,IAAI;IAC3C,IAAIJ,yBAAyB,IAAI,CAACK,gBAChC,OAAOA;IAGT,MAAMH,aAAaG,cAAc,OAAO,CAAC,OAAO;IAChDC,OAAOJ,YAAY;IACnB,OAAOK,wBACLN,uBAAuBC,aACvBA;AAEJ;AAEA,MAAMM,sBAAsB;IAC1B,MAAM;IACN,aAAa;IACb,UAAU;IACV,YAAY;IACZ,WAAW;AACb;AAEO,MAAMC,sBAAsB,CACjCC;IAEA,IAAI,CAACA,YACH;IAGF,IAAIA,cAAcF,qBAChB,OAAOA,mBAAmB,CAACE,WAA+C;IAG5E,OAAOA;AACT;AAEO,eAAeC,oBACpBC,iBAAoC,EACpCC,IAGC;IAED,MAAMC,QAAQf,SAAS;IAEvBO,OAAOM,mBAAmB;IAE1BE,MAAM;IACN,MAAMC,cAAcH,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNE,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMG,gBAAgB,MAAML,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOM,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIL,MAAM,CAAC,MAAM,EAAEI,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMf,mBAAmB,MAAMQ,kBAAkB,gBAAgB;IACjE,MAAMW,uBAAuBC,KAAK,GAAG;IACrClB,OAAOF,kBAAmB;IAC1B,MAAMqB,mBAAmBZ,KAAK,sBAAsB,IAAI;IAExDC,MAAM;IACN,MAAMY,qBAAqB,MAAMC,qBAAqBvB,kBAAkB;QACtE,cAAcqB;IAChB;IACA,MAAM,EAAE,OAAOG,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1CH,mBAAmB,YAAY;IACjCZ,MAAM,yBAAyBc,UAAU,KAAKC;IAM9C,MAAMC,oBAAoBZ,eAAeC;IACzC,MAAMY,uBAAuBH,WAAWC;IACxC,IAAIG,oBAAoBd;IAExB,IAAIY,sBAAsBC,sBAAsB;QAC9CjB,MACE,CAAC,4CAA4C,EAAEI,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEW,oBAAoB,aAAa,YAAY,gBAAgB,EAAEF,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEE,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBb;IAEtB;IAEA,MAAMc,MAAML,WAAWI;IAEvBlB,MAAM,mBAAmBmB;IAEzB,MAAMC,2BAA2BD,MAAMR;IAEvCX,MAAM,4BAA4BoB;IAElC,IAAIT,mBAAmB,GACrBX,MACE,CAAC,kCAAkC,EAAEW,iBAAiB,YAAY,EAAEG,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEH,mBAAmB,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAEA,mBAAmB,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAItL,OAAO;QACL,UAAUA,mBAAmB,QAAQ;QACrC,eAAeO;QACf,YAAYE,eAAe,MAAM,CAC/BT,mBAAmB,MAAM,EACzBH;QAEFW;IACF;AACF;AAEO,eAAeE,+BACpBhC,gBAAwB,EACxBiC,GAEC;IAED,MAAMC,6BACJnC,0BAA0BC;IAC5B,MAAMmC,uBAAuB,MAAMC,kBACjCF;IAEF,IACED,IAAI,cAAc,IACjBA,CAAAA,IAAI,cAAc,CAAC,KAAK,KAAKE,qBAAqB,KAAK,IACtDF,IAAI,cAAc,CAAC,MAAM,KAAKE,qBAAqB,MAAK,GAE1DzC,WACE,mEACA;QACE,UAAUuC,IAAI,cAAc;QAC5B,QAAQE;IACV;IAIJ,OAAO;QACL,YAAYJ,eAAe,MAAM,CAACG,4BAA4Bd,KAAK,GAAG;QACtE,UAAUe;QACV,0BAA0B;QAC1B,WAAW;IACb;AACF;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIC,oBAAoB,qBAAqB,CAACC,wBAC5C;IAEFC,OAAO,CAAC,gCAAgC,EAAEH,UAAU;AACtD;AAUO,SAASI,mBACdC,KAAe,EACfC,UAAqC,CAAC,CAAC;IAEvC,MAAM,EACJC,aAAaC,UAAU,EACvBC,cAAcC,WAAW,EACzBC,cAAcC,OAAO,EACrBC,gBAAgBC,QAAQ,GAAG,CAAC,eAAe,EAC3CC,MAAMD,QAAQ,GAAG,EAAE,EACpB,GAAGR;IAEJ,IAAIG,aACF,MAAM,IAAI7B,MAAM;IAGlB,OAAOyB,MAAM,GAAG,CAAC,CAACW;QAChB,MAAMC,eAAeN,YAAYK;QACjC,IAAI,CAACT,WAAWU,eACd,MAAM,IAAIrC,MACR,CAAC,gBAAgB,EAAEoC,KAAK,eAAe,EAAEC,aAAa,6BAA6B,EAAEF,KAAK;QAI9F,IAAI,CAACF,eACH,OAAOI;QAGT,MAAMC,WAAWD,aAAa,KAAK,CAAC;QACpC,IAAIC,UACF,OAAO,GAAGA,QAAQ,CAAC,EAAE,CAAC,WAAW,GAAG,GAAG,EAAED,aAAa,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,OAAO;QAGvF,OAAO,CAAC,UAAU,EAAEJ,gBAAgBI,aAAa,OAAO,CAAC,OAAO,OAAO;IACzE;AACF;AAUO,SAASE,mCACdC,eAAuD;IAEvD,MAAMC,SAASD,gBAAgB,kBAAkB;IACjD,IAAI,CAACC,UAAU,AAAkB,YAAlB,OAAOA,UAAuB,CAAE,aAAYA,MAAK,GAC9D,OAAO;IAET,MAAMC,QAAQD,OAAO,MAAM;IAC3B,OACEE,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,IAAM,AAAa,YAAb,OAAOA,KAAkB7C,OAAO,QAAQ,CAAC6C;AAEhE;AAEO,SAASC,qBACdL,eAAkD;IAElD,IAAI,CAACA,iBACH;IAGF,MAAM,EAAEM,MAAM,EAAEC,IAAI,EAAE,GAAGP,gBAAgB,kBAAkB;IAC3D,OAAO;QACL,QAAQ;eAAIM;SAAO;QACnB,aACE,AAAkC,YAAlC,OAAON,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;QACxC,GAAIO,OAAO;YAAEA;QAAK,IAAI,CAAC,CAAC;IAC1B;AACF;AAEO,eAAeC,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMN,OACJ,MAAME,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMI,UAA+B;YACnC,QAAQ;gBAACP,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBAAGA,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aAAE;YAChEA;YACA,aACE,AAAuB,YAAvB,OAAOI,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOG;IACT,EAAE,OAAOC,OAAO;QACdF,8BAAW,qCAAqCE;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDP,SACA1C;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAO0C;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNA,QAAQ,MAAM,CAAC,EAAE,GAAG1C;YACpB0C,QAAQ,MAAM,CAAC,EAAE,GAAG1C;SACrB;QACD,GAAI0C,QAAQ,IAAI,GACZ;YACE,MAAMQ,qCACJR,QAAQ,IAAI,EACZ1C;QAEJ,IACA,CAAC,CAAC;IACR;AACF;AAEO,MAAMkD,uCAAuC,CAClDf,MACAnC;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOmC;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMgB,KAAK,KAAK,CAAChB,KAAK,IAAI,GAAGnC;QAC7B,KAAKmD,KAAK,KAAK,CAAChB,KAAK,GAAG,GAAGnC;QAC3B,OAAOmD,KAAK,KAAK,CAAChB,KAAK,KAAK,GAAGnC;QAC/B,QAAQmD,KAAK,KAAK,CAAChB,KAAK,MAAM,GAAGnC;IACnC;AACF"}
1	+ {"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { resolve } from 'node:path';\nimport type { PixelLocateResult } from '@/ai-model/shared/model-locate-result';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n DetailedLocateParam,\n ElementCacheFeature,\n LocateResultElement,\n Rect,\n ScrollParam,\n Size,\n UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n MIDSCENE_REPORT_QUIET,\n globalConfigManager,\n} from '@midscene/shared/env';\nimport {\n createImgBase64ByFormat,\n imageInfoOfBase64,\n} from '@midscene/shared/img';\nimport { getDebug, logReport } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, ifInBrowser } from '@midscene/shared/utils';\nimport { prepareRawScreenshot } from './screenshot-preparation';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nexport { getReportFileName } from './report-file-name';\n\nconst agentDebug = getDebug('agent');\nconst screenshotDataUrlPattern = /^data:image\\/[a-zA-Z0-9.+-]+;base64,/i;\n\nconst inferBase64ImageFormat = (base64Body: string) => {\n if (base64Body.startsWith('iVBORw0KGgo')) {\n return 'png';\n }\n return 'jpeg';\n};\n\nconst normalizeScreenshotBase64 = (screenshotBase64: string) => {\n const trimmedBase64 = screenshotBase64.trim();\n if (screenshotDataUrlPattern.test(trimmedBase64)) {\n return trimmedBase64;\n }\n\n const base64Body = trimmedBase64.replace(/\\s/g, '');\n assert(base64Body, 'screenshotBase64 must include image data');\n return createImgBase64ByFormat(\n inferBase64ImageFormat(base64Body),\n base64Body,\n );\n};\n\nconst legacyScrollTypeMap = {\n once: 'singleAction',\n untilBottom: 'scrollToBottom',\n untilTop: 'scrollToTop',\n untilRight: 'scrollToRight',\n untilLeft: 'scrollToLeft',\n} as const;\n\nexport const normalizeScrollType = (\n scrollType: string \| undefined,\n): ScrollParam['scrollType'] \| undefined => {\n if (!scrollType) {\n return undefined;\n }\n\n if (scrollType in legacyScrollTypeMap) {\n return legacyScrollTypeMap[scrollType as keyof typeof legacyScrollTypeMap];\n }\n\n return scrollType as ScrollParam['scrollType'];\n};\n\nexport async function commonContextParser(\n interfaceInstance: AbstractInterface,\n _opt: {\n uploadServerUrl?: string;\n screenshotShrinkFactor?: number;\n },\n): Promise<UIContext> {\n const debug = getDebug('commonContextParser');\n\n assert(interfaceInstance, 'interfaceInstance is required');\n\n debug('Getting interface description');\n const description = interfaceInstance.describe?.() \|\| '';\n debug('Interface description end');\n\n debug('Uploading test info to server');\n uploadTestInfoToServer({\n testUrl: description,\n serverUrl: _opt.uploadServerUrl,\n });\n debug('UploadTestInfoToServer end');\n\n debug('will get size');\n const interfaceSize = await interfaceInstance.size();\n const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n if ((interfaceSize as unknown as { dpr: number }).dpr) {\n console.warn(\n 'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n );\n }\n\n if (!Number.isFinite(logicalWidth) \|\| !Number.isFinite(logicalHeight)) {\n throw new Error(\n `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n );\n }\n\n if (logicalWidth <= 0 \|\| logicalHeight <= 0) {\n throw new Error(\n `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n );\n }\n\n debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n const screenshotBase64 = await interfaceInstance.screenshotBase64();\n const screenshotCapturedAt = Date.now();\n assert(screenshotBase64!, 'screenshotBase64 is required');\n const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n debug('will get screenshot dimensions');\n const preparedScreenshot = await prepareRawScreenshot(screenshotBase64, {\n shrinkFactor: userShrinkFactor,\n });\n const { width: imgWidth, height: imgHeight } =\n preparedScreenshot.originalSize;\n debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n // Detect orientation mismatch between logical size and screenshot.\n // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n // size() to return portrait dimensions even when the device is landscape.\n // We detect this by comparing aspect ratios and swap if they disagree.\n const logicalIsPortrait = logicalWidth < logicalHeight;\n const screenshotIsPortrait = imgWidth < imgHeight;\n let finalLogicalWidth = logicalWidth;\n let finalLogicalHeight = logicalHeight;\n if (logicalIsPortrait !== screenshotIsPortrait) {\n debug(\n `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n );\n finalLogicalWidth = logicalHeight;\n finalLogicalHeight = logicalWidth;\n }\n\n const dpr = imgWidth / finalLogicalWidth;\n\n debug('calculated dpr:', dpr);\n\n const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n if (userShrinkFactor > 1) {\n debug(\n `Applied screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${preparedScreenshot.shotSize.width}x${preparedScreenshot.shotSize.height})`,\n );\n }\n\n return {\n shotSize: preparedScreenshot.shotSize,\n deprecatedDpr: dpr,\n screenshot: ScreenshotItem.create(\n preparedScreenshot.base64,\n screenshotCapturedAt,\n ),\n shrunkShotToLogicalRatio,\n };\n}\n\nexport async function createScreenshotBoundUIContext(\n screenshotBase64: string,\n opt: {\n screenshotSize?: Size;\n },\n): Promise<UIContext> {\n const normalizedScreenshotBase64 =\n normalizeScreenshotBase64(screenshotBase64);\n const actualScreenshotSize = await imageInfoOfBase64(\n normalizedScreenshotBase64,\n );\n if (\n opt.screenshotSize &&\n (opt.screenshotSize.width !== actualScreenshotSize.width \|\|\n opt.screenshotSize.height !== actualScreenshotSize.height)\n ) {\n agentDebug(\n 'describeElementAtPoint screenshotSize mismatch, use actual size',\n {\n provided: opt.screenshotSize,\n actual: actualScreenshotSize,\n },\n );\n }\n\n return {\n screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),\n shotSize: actualScreenshotSize,\n shrunkShotToLogicalRatio: 1,\n _isFrozen: true,\n };\n}\n\nexport function printReportMsg(filepath: string) {\n if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n return;\n }\n logReport(`Midscene - report file updated: ${filepath}`);\n}\n\ntype NormalizeFilePathsOptions = {\n fileExists?: (path: string) => boolean;\n isInBrowser?: boolean;\n resolvePath?: (path: string) => string;\n wslDistroName?: string;\n cwd?: string;\n};\n\nexport function normalizeFilePaths(\n files: string[],\n options: NormalizeFilePathsOptions = {},\n): string[] {\n const {\n fileExists = existsSync,\n isInBrowser = ifInBrowser,\n resolvePath = resolve,\n wslDistroName = process.env.WSL_DISTRO_NAME,\n cwd = process.cwd(),\n } = options;\n\n if (isInBrowser) {\n throw new Error('File chooser is not supported in browser environment');\n }\n\n return files.map((file) => {\n const absolutePath = resolvePath(file);\n if (!fileExists(absolutePath)) {\n throw new Error(\n `File not found: ${file}. Resolved to: ${absolutePath}. Current working directory: ${cwd}`,\n );\n }\n\n if (!wslDistroName) {\n return absolutePath;\n }\n\n const wslMount = absolutePath.match(/^\\/mnt\\/([a-z])\\//);\n if (wslMount) {\n return `${wslMount[1].toUpperCase()}:\\\\${absolutePath.slice(7).replace(/\\//g, '\\\\')}`;\n }\n\n return `\\\\\\\\wsl$\\\\${wslDistroName}${absolutePath.replace(/\\//g, '\\\\')}`;\n });\n}\n\ntype LocateParamWithMaybeLocatedPixelResult = DetailedLocateParam & {\n locatedPixelResult?: unknown;\n};\n\ntype LocateParamWithLocatedPixelResult = DetailedLocateParam & {\n locatedPixelResult: PixelLocateResult;\n};\n\nexport function ifLocateParamHasLocatedPixelResult(\n planLocateParam: LocateParamWithMaybeLocatedPixelResult,\n): planLocateParam is LocateParamWithLocatedPixelResult {\n const result = planLocateParam.locatedPixelResult;\n if (!result \|\| typeof result !== 'object' \|\| !('center' in result)) {\n return false;\n }\n const point = result.center;\n return (\n Array.isArray(point) &&\n point.length === 2 &&\n point.every((v) => typeof v === 'number' && Number.isFinite(v))\n );\n}\n\nexport function matchElementFromPlan(\n planLocateParam: LocateParamWithLocatedPixelResult,\n): LocateResultElement \| undefined {\n if (!planLocateParam) {\n return undefined;\n }\n\n const { center, rect } = planLocateParam.locatedPixelResult;\n return {\n center: [...center],\n description:\n typeof planLocateParam.prompt === 'string'\n ? planLocateParam.prompt\n : planLocateParam.prompt?.prompt \|\| '',\n ...(rect ? { rect } : {}),\n };\n}\n\nexport async function matchElementFromCache(\n context: {\n taskCache?: TaskCache;\n interfaceInstance: AbstractInterface;\n },\n cacheEntry: ElementCacheFeature \| undefined,\n cachePrompt: TUserPrompt,\n cacheable: boolean \| undefined,\n): Promise<LocateResultElement \| undefined> {\n if (!cacheEntry) {\n return undefined;\n }\n\n if (cacheable === false) {\n cacheDebug('cache disabled for prompt: %s', cachePrompt);\n return undefined;\n }\n\n if (!context.taskCache?.isCacheResultUsed) {\n return undefined;\n }\n\n if (!context.interfaceInstance.rectMatchesCacheFeature) {\n cacheDebug(\n 'interface does not implement rectMatchesCacheFeature, skip cache',\n );\n return undefined;\n }\n\n try {\n const rect =\n await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n const element: LocateResultElement = {\n center: [rect.left + rect.width / 2, rect.top + rect.height / 2],\n rect,\n description:\n typeof cachePrompt === 'string'\n ? cachePrompt\n : cachePrompt.prompt \|\| '',\n };\n\n cacheDebug('cache hit, prompt: %s', cachePrompt);\n return element;\n } catch (error) {\n cacheDebug('rectMatchesCacheFeature error: %s', error);\n return undefined;\n }\n}\n\ndeclare const __VERSION__: string \| undefined;\n\nexport const getMidsceneVersion = (): string => {\n if (typeof __VERSION__ !== 'undefined') {\n return __VERSION__;\n } else if (\n process.env.__VERSION__ &&\n process.env.__VERSION__ !== 'undefined'\n ) {\n return process.env.__VERSION__;\n }\n throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n prompt: TUserPrompt,\n): {\n textPrompt: string;\n multimodalPrompt?: TMultimodalPrompt;\n} => {\n if (typeof prompt === 'string') {\n return {\n textPrompt: prompt,\n multimodalPrompt: undefined,\n };\n }\n return {\n textPrompt: prompt.prompt,\n multimodalPrompt: prompt.images\n ? {\n images: prompt.images,\n convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n }\n : undefined,\n };\n};\n\nexport const transformLogicalElementToScreenshot = (\n element: LocateResultElement,\n shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n if (shrunkShotToLogicalRatio === 1) {\n return element;\n }\n\n return {\n ...element,\n center: [\n element.center[0] * shrunkShotToLogicalRatio,\n element.center[1] * shrunkShotToLogicalRatio,\n ],\n ...(element.rect\n ? {\n rect: transformLogicalRectToScreenshotRect(\n element.rect,\n shrunkShotToLogicalRatio,\n ),\n }\n : {}),\n };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n rect: Rect,\n shrunkShotToLogicalRatio: number,\n): Rect => {\n if (shrunkShotToLogicalRatio === 1) {\n return rect;\n }\n\n return {\n ...rect,\n left: Math.round(rect.left * shrunkShotToLogicalRatio),\n top: Math.round(rect.top * shrunkShotToLogicalRatio),\n width: Math.round(rect.width * shrunkShotToLogicalRatio),\n height: Math.round(rect.height * shrunkShotToLogicalRatio),\n };\n};\n"],"names":["agentDebug","getDebug","screenshotDataUrlPattern","inferBase64ImageFormat","base64Body","normalizeScreenshotBase64","screenshotBase64","trimmedBase64","assert","createImgBase64ByFormat","legacyScrollTypeMap","normalizeScrollType","scrollType","commonContextParser","interfaceInstance","_opt","debug","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotCapturedAt","Date","userShrinkFactor","preparedScreenshot","prepareRawScreenshot","imgWidth","imgHeight","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","dpr","shrunkShotToLogicalRatio","ScreenshotItem","createScreenshotBoundUIContext","opt","normalizedScreenshotBase64","actualScreenshotSize","imageInfoOfBase64","printReportMsg","filepath","globalConfigManager","MIDSCENE_REPORT_QUIET","logReport","normalizeFilePaths","files","options","fileExists","existsSync","isInBrowser","ifInBrowser","resolvePath","resolve","wslDistroName","process","cwd","file","absolutePath","wslMount","ifLocateParamHasLocatedPixelResult","planLocateParam","result","point","Array","v","matchElementFromPlan","center","rect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","element","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect","Math"],"mappings":";;;;;;;;;;;AAiCA,MAAMA,aAAaC,SAAS;AAC5B,MAAMC,2BAA2B;AAEjC,MAAMC,yBAAyB,CAACC;IAC9B,IAAIA,WAAW,UAAU,CAAC,gBACxB,OAAO;IAET,OAAO;AACT;AAEA,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,gBAAgBD,iBAAiB,IAAI;IAC3C,IAAIJ,yBAAyB,IAAI,CAACK,gBAChC,OAAOA;IAGT,MAAMH,aAAaG,cAAc,OAAO,CAAC,OAAO;IAChDC,OAAOJ,YAAY;IACnB,OAAOK,wBACLN,uBAAuBC,aACvBA;AAEJ;AAEA,MAAMM,sBAAsB;IAC1B,MAAM;IACN,aAAa;IACb,UAAU;IACV,YAAY;IACZ,WAAW;AACb;AAEO,MAAMC,sBAAsB,CACjCC;IAEA,IAAI,CAACA,YACH;IAGF,IAAIA,cAAcF,qBAChB,OAAOA,mBAAmB,CAACE,WAA+C;IAG5E,OAAOA;AACT;AAEO,eAAeC,oBACpBC,iBAAoC,EACpCC,IAGC;IAED,MAAMC,QAAQf,SAAS;IAEvBO,OAAOM,mBAAmB;IAE1BE,MAAM;IACN,MAAMC,cAAcH,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNE,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMG,gBAAgB,MAAML,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOM,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIL,MAAM,CAAC,MAAM,EAAEI,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMf,mBAAmB,MAAMQ,kBAAkB,gBAAgB;IACjE,MAAMW,uBAAuBC,KAAK,GAAG;IACrClB,OAAOF,kBAAmB;IAC1B,MAAMqB,mBAAmBZ,KAAK,sBAAsB,IAAI;IAExDC,MAAM;IACN,MAAMY,qBAAqB,MAAMC,qBAAqBvB,kBAAkB;QACtE,cAAcqB;IAChB;IACA,MAAM,EAAE,OAAOG,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1CH,mBAAmB,YAAY;IACjCZ,MAAM,yBAAyBc,UAAU,KAAKC;IAM9C,MAAMC,oBAAoBZ,eAAeC;IACzC,MAAMY,uBAAuBH,WAAWC;IACxC,IAAIG,oBAAoBd;IAExB,IAAIY,sBAAsBC,sBAAsB;QAC9CjB,MACE,CAAC,4CAA4C,EAAEI,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEW,oBAAoB,aAAa,YAAY,gBAAgB,EAAEF,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEE,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBb;IAEtB;IAEA,MAAMc,MAAML,WAAWI;IAEvBlB,MAAM,mBAAmBmB;IAEzB,MAAMC,2BAA2BD,MAAMR;IAEvCX,MAAM,4BAA4BoB;IAElC,IAAIT,mBAAmB,GACrBX,MACE,CAAC,kCAAkC,EAAEW,iBAAiB,YAAY,EAAEG,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEH,mBAAmB,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAEA,mBAAmB,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAItL,OAAO;QACL,UAAUA,mBAAmB,QAAQ;QACrC,eAAeO;QACf,YAAYE,eAAe,MAAM,CAC/BT,mBAAmB,MAAM,EACzBH;QAEFW;IACF;AACF;AAEO,eAAeE,+BACpBhC,gBAAwB,EACxBiC,GAEC;IAED,MAAMC,6BACJnC,0BAA0BC;IAC5B,MAAMmC,uBAAuB,MAAMC,kBACjCF;IAEF,IACED,IAAI,cAAc,IACjBA,CAAAA,IAAI,cAAc,CAAC,KAAK,KAAKE,qBAAqB,KAAK,IACtDF,IAAI,cAAc,CAAC,MAAM,KAAKE,qBAAqB,MAAK,GAE1DzC,WACE,mEACA;QACE,UAAUuC,IAAI,cAAc;QAC5B,QAAQE;IACV;IAIJ,OAAO;QACL,YAAYJ,eAAe,MAAM,CAACG,4BAA4Bd,KAAK,GAAG;QACtE,UAAUe;QACV,0BAA0B;QAC1B,WAAW;IACb;AACF;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIC,oBAAoB,qBAAqB,CAACC,wBAC5C;IAEFC,UAAU,CAAC,gCAAgC,EAAEH,UAAU;AACzD;AAUO,SAASI,mBACdC,KAAe,EACfC,UAAqC,CAAC,CAAC;IAEvC,MAAM,EACJC,aAAaC,UAAU,EACvBC,cAAcC,WAAW,EACzBC,cAAcC,OAAO,EACrBC,gBAAgBC,QAAQ,GAAG,CAAC,eAAe,EAC3CC,MAAMD,QAAQ,GAAG,EAAE,EACpB,GAAGR;IAEJ,IAAIG,aACF,MAAM,IAAI7B,MAAM;IAGlB,OAAOyB,MAAM,GAAG,CAAC,CAACW;QAChB,MAAMC,eAAeN,YAAYK;QACjC,IAAI,CAACT,WAAWU,eACd,MAAM,IAAIrC,MACR,CAAC,gBAAgB,EAAEoC,KAAK,eAAe,EAAEC,aAAa,6BAA6B,EAAEF,KAAK;QAI9F,IAAI,CAACF,eACH,OAAOI;QAGT,MAAMC,WAAWD,aAAa,KAAK,CAAC;QACpC,IAAIC,UACF,OAAO,GAAGA,QAAQ,CAAC,EAAE,CAAC,WAAW,GAAG,GAAG,EAAED,aAAa,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,OAAO;QAGvF,OAAO,CAAC,UAAU,EAAEJ,gBAAgBI,aAAa,OAAO,CAAC,OAAO,OAAO;IACzE;AACF;AAUO,SAASE,mCACdC,eAAuD;IAEvD,MAAMC,SAASD,gBAAgB,kBAAkB;IACjD,IAAI,CAACC,UAAU,AAAkB,YAAlB,OAAOA,UAAuB,CAAE,aAAYA,MAAK,GAC9D,OAAO;IAET,MAAMC,QAAQD,OAAO,MAAM;IAC3B,OACEE,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,IAAM,AAAa,YAAb,OAAOA,KAAkB7C,OAAO,QAAQ,CAAC6C;AAEhE;AAEO,SAASC,qBACdL,eAAkD;IAElD,IAAI,CAACA,iBACH;IAGF,MAAM,EAAEM,MAAM,EAAEC,IAAI,EAAE,GAAGP,gBAAgB,kBAAkB;IAC3D,OAAO;QACL,QAAQ;eAAIM;SAAO;QACnB,aACE,AAAkC,YAAlC,OAAON,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;QACxC,GAAIO,OAAO;YAAEA;QAAK,IAAI,CAAC,CAAC;IAC1B;AACF;AAEO,eAAeC,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMN,OACJ,MAAME,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMI,UAA+B;YACnC,QAAQ;gBAACP,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBAAGA,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aAAE;YAChEA;YACA,aACE,AAAuB,YAAvB,OAAOI,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOG;IACT,EAAE,OAAOC,OAAO;QACdF,8BAAW,qCAAqCE;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDP,SACA1C;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAO0C;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNA,QAAQ,MAAM,CAAC,EAAE,GAAG1C;YACpB0C,QAAQ,MAAM,CAAC,EAAE,GAAG1C;SACrB;QACD,GAAI0C,QAAQ,IAAI,GACZ;YACE,MAAMQ,qCACJR,QAAQ,IAAI,EACZ1C;QAEJ,IACA,CAAC,CAAC;IACR;AACF;AAEO,MAAMkD,uCAAuC,CAClDf,MACAnC;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOmC;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMgB,KAAK,KAAK,CAAChB,KAAK,IAAI,GAAGnC;QAC7B,KAAKmD,KAAK,KAAK,CAAChB,KAAK,GAAG,GAAGnC;QAC3B,OAAOmD,KAAK,KAAK,CAAChB,KAAK,KAAK,GAAGnC;QAC/B,QAAQmD,KAAK,KAAK,CAAChB,KAAK,MAAM,GAAGnC;IACnC;AACF"}

package/dist/es/ai-model/models/auto-glm/parser.mjs CHANGED Viewed

@@ -23,7 +23,7 @@ function parseAutoGLMPlanningAction(response) {
         }
         if (trimmedResponse.startsWith('finish(message=')) {
             let message = extractValueAfter(trimmedResponse, 'finish(message="');
-            if (message.endsWith(')')) message = message.slice(0, -1);
+            if (!trimmedResponse.endsWith('")') && message.endsWith(')')) message = message.slice(0, -1);
             return {
                 _metadata: 'finish',
                 message,

package/dist/es/ai-model/models/auto-glm/parser.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/models/auto-glm/parser.mjs","sources":["../../../../../src/ai-model/models/auto-glm/parser.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport type { AutoGLMParsedAction } from './actions';\n\nconst debug = getDebug('auto-glm-parser');\n\n// Do not rely on regex extraction here; regex can fail on malformed input.\n// Bad Case: finish(message=\"Finished! Now There is a contact whose name is \\\"Tom\\\" in the list.\")\nexport const extractValueAfter = (src: string, key: string): string => {\n const idx = src.indexOf(key);\n if (idx === -1) {\n throw new Error(`Missing key ${key} in action payload ${src}`);\n }\n let rest = src.slice(idx + key.length).trim();\n if (rest.endsWith('\")')) {\n rest = rest.slice(0, -2);\n }\n return rest;\n};\n\nexport function parseAutoGLMPlanningAction(response: {\n think: string;\n content: string;\n}): AutoGLMParsedAction {\n debug('Parsing action:', response);\n let trimmedResponse = '';\n try {\n trimmedResponse = response.content.trim();\n\n if (\n trimmedResponse.startsWith('do(action=\"Type\"') \|\|\n trimmedResponse.startsWith('do(action=\"Type_Name\"')\n ) {\n const text = extractValueAfter(trimmedResponse, 'text=\"');\n return {\n _metadata: 'do',\n action: 'Type',\n text,\n think: response.think,\n } as AutoGLMParsedAction;\n }\n\n if (trimmedResponse.startsWith('finish(message=')) {\n let message = extractValueAfter(trimmedResponse, 'finish(message=\"');\n if (message.endsWith(')')) message = message.slice(0, -1);\n return {\n _metadata: 'finish',\n message,\n think: response.think,\n } as AutoGLMParsedAction;\n }\n\n if (trimmedResponse.startsWith('do(')) {\n const actionMatch = trimmedResponse.match(/do\\(action=\"([^\"]+)\"/);\n if (!actionMatch)\n throw new Error(\n `Failed to extract action type from do() call; raw=\"${trimmedResponse}\"`,\n );\n const actionType = actionMatch[1];\n\n const baseAction = { _metadata: 'do' as const, think: response.think };\n switch (actionType) {\n case 'Tap': {\n const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n if (!elementMatch)\n throw new Error(\n `Failed to extract element coordinates for Tap; raw=\"${trimmedResponse}\"`,\n );\n return {\n ...baseAction,\n action: 'Tap',\n element: [Number(elementMatch[1]), Number(elementMatch[2])],\n } as AutoGLMParsedAction;\n }\n case 'Double Tap': {\n const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n if (!elementMatch)\n throw new Error(\n `Failed to extract element coordinates for Double Tap; raw=\"${trimmedResponse}\"`,\n );\n return {\n ...baseAction,\n action: 'Double Tap',\n element: [Number(elementMatch[1]), Number(elementMatch[2])],\n } as AutoGLMParsedAction;\n }\n case 'Swipe': {\n const startMatch = trimmedResponse.match(/start=\\[(\\d+),(\\d+)\\]/);\n const endMatch = trimmedResponse.match(/end=\\[(\\d+),(\\d+)\\]/);\n if (!startMatch \|\| !endMatch)\n throw new Error(\n `Failed to extract start/end coordinates for Swipe; raw=\"${trimmedResponse}\"`,\n );\n return {\n ...baseAction,\n action: 'Swipe',\n start: [Number(startMatch[1]), Number(startMatch[2])],\n end: [Number(endMatch[1]), Number(endMatch[2])],\n } as AutoGLMParsedAction;\n }\n case 'Long Press': {\n const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n if (!elementMatch)\n throw new Error(\n `Failed to extract element coordinates for Long Press; raw=\"${trimmedResponse}\"`,\n );\n return {\n ...baseAction,\n action: 'Long Press',\n element: [Number(elementMatch[1]), Number(elementMatch[2])],\n } as AutoGLMParsedAction;\n }\n case 'Launch': {\n const app = extractValueAfter(trimmedResponse, 'app=\"');\n return {\n ...baseAction,\n action: 'Launch',\n app,\n } as AutoGLMParsedAction;\n }\n case 'Back': {\n return { ...baseAction, action: 'Back' } as AutoGLMParsedAction;\n }\n case 'Home': {\n return { ...baseAction, action: 'Home' } as AutoGLMParsedAction;\n }\n case 'Wait': {\n const durationMatch = trimmedResponse.match(\n /duration=(?:[\"\\[])?(\\d+)/,\n );\n if (!durationMatch) {\n throw new Error(\n `Failed to extract duration for Wait; raw=\"${trimmedResponse}\"`,\n );\n }\n const seconds = Number.parseInt(durationMatch[1], 10);\n const durationMs = seconds * 1000;\n return {\n ...baseAction,\n action: 'Wait',\n durationMs,\n } as AutoGLMParsedAction;\n }\n case 'Interact': {\n return { ...baseAction, action: 'Interact' } as AutoGLMParsedAction;\n }\n case 'Call_API': {\n const instruction = extractValueAfter(\n trimmedResponse,\n 'instruction=\"',\n );\n return {\n ...baseAction,\n action: 'Call_API',\n instruction,\n } as AutoGLMParsedAction;\n }\n case 'Take_over': {\n const message = extractValueAfter(trimmedResponse, 'message=\"');\n return {\n ...baseAction,\n action: 'Take_over',\n message,\n } as AutoGLMParsedAction;\n }\n case 'Note': {\n const message = extractValueAfter(trimmedResponse, 'message=\"');\n return {\n ...baseAction,\n action: 'Note',\n message,\n } as AutoGLMParsedAction;\n }\n default:\n throw new Error(\n `Unknown action type: ${actionType}; raw=\"${trimmedResponse}\"`,\n );\n }\n }\n throw new Error(`Failed to parse action: ${trimmedResponse}`);\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n throw new Error(\n `Failed to parse action: ${errorMessage}; raw=\"${trimmedResponse}\"`,\n );\n }\n}\n\nexport function parseAutoGLMResponse(content: string): {\n think: string;\n content: string;\n} {\n let parsedResponse: { think: string; content: string };\n\n if (content.includes('finish(message=')) {\n const parts = content.split('finish(message=');\n const think = parts[0].trim();\n const actionContent = `finish(message=${parts[1]}`;\n parsedResponse = { think, content: actionContent };\n } else if (content.includes('do(action=')) {\n const parts = content.split('do(action=');\n const think = parts[0].trim();\n const actionContent = `do(action=${parts[1]}`;\n parsedResponse = { think, content: actionContent };\n } else if (content.includes('<answer>')) {\n const parts = content.split('<answer>');\n const think = parts[0]\n .replace(/<think>/g, '')\n .replace(/<\\/think>/g, '')\n .trim();\n const actionContent = parts[1].replace(/<\\/answer>/g, '').trim();\n parsedResponse = { think, content: actionContent };\n } else {\n parsedResponse = { think: '', content };\n }\n\n debug('autoGLM rawResponse:', content);\n debug('thinking in response:', parsedResponse.think);\n debug('action in response:', parsedResponse.content);\n return parsedResponse;\n}\n\nexport function parseAutoGLMPlanningResponse(content: string): {\n response: ReturnType<typeof parseAutoGLMResponse>;\n action: AutoGLMParsedAction;\n} {\n const response = parseAutoGLMResponse(content);\n const action = parseAutoGLMPlanningAction(response);\n return { response, action };\n}\n"],"names":["debug","getDebug","extractValueAfter","src","key","idx","Error","rest","parseAutoGLMPlanningAction","response","trimmedResponse","text","message","actionMatch","actionType","baseAction","elementMatch","Number","startMatch","endMatch","app","durationMatch","seconds","durationMs","instruction","error","errorMessage","String","parseAutoGLMResponse","content","parsedResponse","parts","think","actionContent","parseAutoGLMPlanningResponse","action"],"mappings":";AAGA,MAAMA,QAAQC,SAAS;AAIhB,MAAMC,oBAAoB,CAACC,KAAaC;IAC7C,MAAMC,MAAMF,IAAI,OAAO,CAACC;IACxB,IAAIC,AAAQ,OAARA,KACF,MAAM,IAAIC,MAAM,CAAC,YAAY,EAAEF,IAAI,mBAAmB,EAAED,KAAK;IAE/D,IAAII,OAAOJ,IAAI,KAAK,CAACE,MAAMD,IAAI,MAAM,EAAE,IAAI;IAC3C,IAAIG,KAAK,QAAQ,CAAC,OAChBA,OAAOA,KAAK,KAAK,CAAC,GAAG;IAEvB,OAAOA;AACT;AAEO,SAASC,2BAA2BC,QAG1C;IACCT,MAAM,mBAAmBS;IACzB,IAAIC,kBAAkB;IACtB,IAAI;QACFA,kBAAkBD,SAAS,OAAO,CAAC,IAAI;QAEvC,IACEC,gBAAgB,UAAU,CAAC,uBAC3BA,gBAAgB,UAAU,CAAC,0BAC3B;YACA,MAAMC,OAAOT,kBAAkBQ,iBAAiB;YAChD,OAAO;gBACL,WAAW;gBACX,QAAQ;gBACRC;gBACA,OAAOF,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,oBAAoB;YACjD,IAAIE,UAAUV,kBAAkBQ,iBAAiB;YACjD,IAAIE,QAAQ,QAAQ,CAAC,MAAMA,UAAUA,QAAQ,KAAK,CAAC,GAAG;YACtD,OAAO;gBACL,WAAW;gBACXA;gBACA,OAAOH,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,QAAQ;YACrC,MAAMG,cAAcH,gBAAgB,KAAK,CAAC;YAC1C,IAAI,CAACG,aACH,MAAM,IAAIP,MACR,CAAC,mDAAmD,EAAEI,gBAAgB,CAAC,CAAC;YAE5E,MAAMI,aAAaD,WAAW,CAAC,EAAE;YAEjC,MAAME,aAAa;gBAAE,WAAW;gBAAe,OAAON,SAAS,KAAK;YAAC;YACrE,OAAQK;gBACN,KAAK;oBAAO;wBACV,MAAME,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,oDAAoD,EAAEI,gBAAgB,CAAC,CAAC;wBAE7E,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMA,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAS;wBACZ,MAAME,aAAaR,gBAAgB,KAAK,CAAC;wBACzC,MAAMS,WAAWT,gBAAgB,KAAK,CAAC;wBACvC,IAAI,CAACQ,cAAc,CAACC,UAClB,MAAM,IAAIb,MACR,CAAC,wDAAwD,EAAEI,gBAAgB,CAAC,CAAC;wBAEjF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,OAAO;gCAACE,OAAOC,UAAU,CAAC,EAAE;gCAAGD,OAAOC,UAAU,CAAC,EAAE;6BAAE;4BACrD,KAAK;gCAACD,OAAOE,QAAQ,CAAC,EAAE;gCAAGF,OAAOE,QAAQ,CAAC,EAAE;6BAAE;wBACjD;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMH,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAU;wBACb,MAAMI,MAAMlB,kBAAkBQ,iBAAiB;wBAC/C,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRK;wBACF;oBACF;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGL,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBACH,OAAO;wBAAE,GAAGA,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBAAQ;wBACX,MAAMM,gBAAgBX,gBAAgB,KAAK,CACzC;wBAEF,IAAI,CAACW,eACH,MAAM,IAAIf,MACR,CAAC,0CAA0C,EAAEI,gBAAgB,CAAC,CAAC;wBAGnE,MAAMY,UAAUL,OAAO,QAAQ,CAACI,aAAa,CAAC,EAAE,EAAE;wBAClD,MAAME,aAAaD,AAAU,OAAVA;wBACnB,OAAO;4BACL,GAAGP,UAAU;4BACb,QAAQ;4BACRQ;wBACF;oBACF;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGR,UAAU;wBAAE,QAAQ;oBAAW;gBAE7C,KAAK;oBAAY;wBACf,MAAMS,cAActB,kBAClBQ,iBACA;wBAEF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRS;wBACF;oBACF;gBACA,KAAK;oBAAa;wBAChB,MAAMZ,UAAUV,kBAAkBQ,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA,KAAK;oBAAQ;wBACX,MAAMA,UAAUV,kBAAkBQ,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA;oBACE,MAAM,IAAIN,MACR,CAAC,qBAAqB,EAAEQ,WAAW,OAAO,EAAEJ,gBAAgB,CAAC,CAAC;YAEpE;QACF;QACA,MAAM,IAAIJ,MAAM,CAAC,wBAAwB,EAAEI,iBAAiB;IAC9D,EAAE,OAAOe,OAAO;QACd,MAAMC,eAAeD,iBAAiBnB,QAAQmB,MAAM,OAAO,GAAGE,OAAOF;QACrE,MAAM,IAAInB,MACR,CAAC,wBAAwB,EAAEoB,aAAa,OAAO,EAAEhB,gBAAgB,CAAC,CAAC;IAEvE;AACF;AAEO,SAASkB,qBAAqBC,OAAe;IAIlD,IAAIC;IAEJ,IAAID,QAAQ,QAAQ,CAAC,oBAAoB;QACvC,MAAME,QAAQF,QAAQ,KAAK,CAAC;QAC5B,MAAMG,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,eAAe,EAAEF,KAAK,CAAC,EAAE,EAAE;QAClDD,iBAAiB;YAAEE;YAAO,SAASC;QAAc;IACnD,OAAO,IAAIJ,QAAQ,QAAQ,CAAC,eAAe;QACzC,MAAME,QAAQF,QAAQ,KAAK,CAAC;QAC5B,MAAMG,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,UAAU,EAAEF,KAAK,CAAC,EAAE,EAAE;QAC7CD,iBAAiB;YAAEE;YAAO,SAASC;QAAc;IACnD,OAAO,IAAIJ,QAAQ,QAAQ,CAAC,aAAa;QACvC,MAAME,QAAQF,QAAQ,KAAK,CAAC;QAC5B,MAAMG,QAAQD,KAAK,CAAC,EAAE,CACnB,OAAO,CAAC,YAAY,IACpB,OAAO,CAAC,cAAc,IACtB,IAAI;QACP,MAAME,gBAAgBF,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,IAAI;QAC9DD,iBAAiB;YAAEE;YAAO,SAASC;QAAc;IACnD,OACEH,iBAAiB;QAAE,OAAO;QAAID;IAAQ;IAGxC7B,MAAM,wBAAwB6B;IAC9B7B,MAAM,yBAAyB8B,eAAe,KAAK;IACnD9B,MAAM,uBAAuB8B,eAAe,OAAO;IACnD,OAAOA;AACT;AAEO,SAASI,6BAA6BL,OAAe;IAI1D,MAAMpB,WAAWmB,qBAAqBC;IACtC,MAAMM,SAAS3B,2BAA2BC;IAC1C,OAAO;QAAEA;QAAU0B;IAAO;AAC5B"}
1	+ {"version":3,"file":"ai-model/models/auto-glm/parser.mjs","sources":["../../../../../src/ai-model/models/auto-glm/parser.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport type { AutoGLMParsedAction } from './actions';\n\nconst debug = getDebug('auto-glm-parser');\n\n// Do not rely on regex extraction here; regex can fail on malformed input.\n// Bad Case: finish(message=\"Finished! Now There is a contact whose name is \\\"Tom\\\" in the list.\")\nexport const extractValueAfter = (src: string, key: string): string => {\n const idx = src.indexOf(key);\n if (idx === -1) {\n throw new Error(`Missing key ${key} in action payload ${src}`);\n }\n let rest = src.slice(idx + key.length).trim();\n if (rest.endsWith('\")')) {\n rest = rest.slice(0, -2);\n }\n return rest;\n};\n\nexport function parseAutoGLMPlanningAction(response: {\n think: string;\n content: string;\n}): AutoGLMParsedAction {\n debug('Parsing action:', response);\n let trimmedResponse = '';\n try {\n trimmedResponse = response.content.trim();\n\n if (\n trimmedResponse.startsWith('do(action=\"Type\"') \|\|\n trimmedResponse.startsWith('do(action=\"Type_Name\"')\n ) {\n const text = extractValueAfter(trimmedResponse, 'text=\"');\n return {\n _metadata: 'do',\n action: 'Type',\n text,\n think: response.think,\n } as AutoGLMParsedAction;\n }\n\n if (trimmedResponse.startsWith('finish(message=')) {\n let message = extractValueAfter(trimmedResponse, 'finish(message=\"');\n if (!trimmedResponse.endsWith('\")') && message.endsWith(')')) {\n message = message.slice(0, -1);\n }\n return {\n _metadata: 'finish',\n message,\n think: response.think,\n } as AutoGLMParsedAction;\n }\n\n if (trimmedResponse.startsWith('do(')) {\n const actionMatch = trimmedResponse.match(/do\\(action=\"([^\"]+)\"/);\n if (!actionMatch)\n throw new Error(\n `Failed to extract action type from do() call; raw=\"${trimmedResponse}\"`,\n );\n const actionType = actionMatch[1];\n\n const baseAction = { _metadata: 'do' as const, think: response.think };\n switch (actionType) {\n case 'Tap': {\n const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n if (!elementMatch)\n throw new Error(\n `Failed to extract element coordinates for Tap; raw=\"${trimmedResponse}\"`,\n );\n return {\n ...baseAction,\n action: 'Tap',\n element: [Number(elementMatch[1]), Number(elementMatch[2])],\n } as AutoGLMParsedAction;\n }\n case 'Double Tap': {\n const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n if (!elementMatch)\n throw new Error(\n `Failed to extract element coordinates for Double Tap; raw=\"${trimmedResponse}\"`,\n );\n return {\n ...baseAction,\n action: 'Double Tap',\n element: [Number(elementMatch[1]), Number(elementMatch[2])],\n } as AutoGLMParsedAction;\n }\n case 'Swipe': {\n const startMatch = trimmedResponse.match(/start=\\[(\\d+),(\\d+)\\]/);\n const endMatch = trimmedResponse.match(/end=\\[(\\d+),(\\d+)\\]/);\n if (!startMatch \|\| !endMatch)\n throw new Error(\n `Failed to extract start/end coordinates for Swipe; raw=\"${trimmedResponse}\"`,\n );\n return {\n ...baseAction,\n action: 'Swipe',\n start: [Number(startMatch[1]), Number(startMatch[2])],\n end: [Number(endMatch[1]), Number(endMatch[2])],\n } as AutoGLMParsedAction;\n }\n case 'Long Press': {\n const elementMatch = trimmedResponse.match(/element=\\[(\\d+),(\\d+)\\]/);\n if (!elementMatch)\n throw new Error(\n `Failed to extract element coordinates for Long Press; raw=\"${trimmedResponse}\"`,\n );\n return {\n ...baseAction,\n action: 'Long Press',\n element: [Number(elementMatch[1]), Number(elementMatch[2])],\n } as AutoGLMParsedAction;\n }\n case 'Launch': {\n const app = extractValueAfter(trimmedResponse, 'app=\"');\n return {\n ...baseAction,\n action: 'Launch',\n app,\n } as AutoGLMParsedAction;\n }\n case 'Back': {\n return { ...baseAction, action: 'Back' } as AutoGLMParsedAction;\n }\n case 'Home': {\n return { ...baseAction, action: 'Home' } as AutoGLMParsedAction;\n }\n case 'Wait': {\n const durationMatch = trimmedResponse.match(\n /duration=(?:[\"\\[])?(\\d+)/,\n );\n if (!durationMatch) {\n throw new Error(\n `Failed to extract duration for Wait; raw=\"${trimmedResponse}\"`,\n );\n }\n const seconds = Number.parseInt(durationMatch[1], 10);\n const durationMs = seconds * 1000;\n return {\n ...baseAction,\n action: 'Wait',\n durationMs,\n } as AutoGLMParsedAction;\n }\n case 'Interact': {\n return { ...baseAction, action: 'Interact' } as AutoGLMParsedAction;\n }\n case 'Call_API': {\n const instruction = extractValueAfter(\n trimmedResponse,\n 'instruction=\"',\n );\n return {\n ...baseAction,\n action: 'Call_API',\n instruction,\n } as AutoGLMParsedAction;\n }\n case 'Take_over': {\n const message = extractValueAfter(trimmedResponse, 'message=\"');\n return {\n ...baseAction,\n action: 'Take_over',\n message,\n } as AutoGLMParsedAction;\n }\n case 'Note': {\n const message = extractValueAfter(trimmedResponse, 'message=\"');\n return {\n ...baseAction,\n action: 'Note',\n message,\n } as AutoGLMParsedAction;\n }\n default:\n throw new Error(\n `Unknown action type: ${actionType}; raw=\"${trimmedResponse}\"`,\n );\n }\n }\n throw new Error(`Failed to parse action: ${trimmedResponse}`);\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error);\n throw new Error(\n `Failed to parse action: ${errorMessage}; raw=\"${trimmedResponse}\"`,\n );\n }\n}\n\nexport function parseAutoGLMResponse(content: string): {\n think: string;\n content: string;\n} {\n let parsedResponse: { think: string; content: string };\n\n if (content.includes('finish(message=')) {\n const parts = content.split('finish(message=');\n const think = parts[0].trim();\n const actionContent = `finish(message=${parts[1]}`;\n parsedResponse = { think, content: actionContent };\n } else if (content.includes('do(action=')) {\n const parts = content.split('do(action=');\n const think = parts[0].trim();\n const actionContent = `do(action=${parts[1]}`;\n parsedResponse = { think, content: actionContent };\n } else if (content.includes('<answer>')) {\n const parts = content.split('<answer>');\n const think = parts[0]\n .replace(/<think>/g, '')\n .replace(/<\\/think>/g, '')\n .trim();\n const actionContent = parts[1].replace(/<\\/answer>/g, '').trim();\n parsedResponse = { think, content: actionContent };\n } else {\n parsedResponse = { think: '', content };\n }\n\n debug('autoGLM rawResponse:', content);\n debug('thinking in response:', parsedResponse.think);\n debug('action in response:', parsedResponse.content);\n return parsedResponse;\n}\n\nexport function parseAutoGLMPlanningResponse(content: string): {\n response: ReturnType<typeof parseAutoGLMResponse>;\n action: AutoGLMParsedAction;\n} {\n const response = parseAutoGLMResponse(content);\n const action = parseAutoGLMPlanningAction(response);\n return { response, action };\n}\n"],"names":["debug","getDebug","extractValueAfter","src","key","idx","Error","rest","parseAutoGLMPlanningAction","response","trimmedResponse","text","message","actionMatch","actionType","baseAction","elementMatch","Number","startMatch","endMatch","app","durationMatch","seconds","durationMs","instruction","error","errorMessage","String","parseAutoGLMResponse","content","parsedResponse","parts","think","actionContent","parseAutoGLMPlanningResponse","action"],"mappings":";AAGA,MAAMA,QAAQC,SAAS;AAIhB,MAAMC,oBAAoB,CAACC,KAAaC;IAC7C,MAAMC,MAAMF,IAAI,OAAO,CAACC;IACxB,IAAIC,AAAQ,OAARA,KACF,MAAM,IAAIC,MAAM,CAAC,YAAY,EAAEF,IAAI,mBAAmB,EAAED,KAAK;IAE/D,IAAII,OAAOJ,IAAI,KAAK,CAACE,MAAMD,IAAI,MAAM,EAAE,IAAI;IAC3C,IAAIG,KAAK,QAAQ,CAAC,OAChBA,OAAOA,KAAK,KAAK,CAAC,GAAG;IAEvB,OAAOA;AACT;AAEO,SAASC,2BAA2BC,QAG1C;IACCT,MAAM,mBAAmBS;IACzB,IAAIC,kBAAkB;IACtB,IAAI;QACFA,kBAAkBD,SAAS,OAAO,CAAC,IAAI;QAEvC,IACEC,gBAAgB,UAAU,CAAC,uBAC3BA,gBAAgB,UAAU,CAAC,0BAC3B;YACA,MAAMC,OAAOT,kBAAkBQ,iBAAiB;YAChD,OAAO;gBACL,WAAW;gBACX,QAAQ;gBACRC;gBACA,OAAOF,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,oBAAoB;YACjD,IAAIE,UAAUV,kBAAkBQ,iBAAiB;YACjD,IAAI,CAACA,gBAAgB,QAAQ,CAAC,SAASE,QAAQ,QAAQ,CAAC,MACtDA,UAAUA,QAAQ,KAAK,CAAC,GAAG;YAE7B,OAAO;gBACL,WAAW;gBACXA;gBACA,OAAOH,SAAS,KAAK;YACvB;QACF;QAEA,IAAIC,gBAAgB,UAAU,CAAC,QAAQ;YACrC,MAAMG,cAAcH,gBAAgB,KAAK,CAAC;YAC1C,IAAI,CAACG,aACH,MAAM,IAAIP,MACR,CAAC,mDAAmD,EAAEI,gBAAgB,CAAC,CAAC;YAE5E,MAAMI,aAAaD,WAAW,CAAC,EAAE;YAEjC,MAAME,aAAa;gBAAE,WAAW;gBAAe,OAAON,SAAS,KAAK;YAAC;YACrE,OAAQK;gBACN,KAAK;oBAAO;wBACV,MAAME,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,oDAAoD,EAAEI,gBAAgB,CAAC,CAAC;wBAE7E,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMA,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAS;wBACZ,MAAME,aAAaR,gBAAgB,KAAK,CAAC;wBACzC,MAAMS,WAAWT,gBAAgB,KAAK,CAAC;wBACvC,IAAI,CAACQ,cAAc,CAACC,UAClB,MAAM,IAAIb,MACR,CAAC,wDAAwD,EAAEI,gBAAgB,CAAC,CAAC;wBAEjF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,OAAO;gCAACE,OAAOC,UAAU,CAAC,EAAE;gCAAGD,OAAOC,UAAU,CAAC,EAAE;6BAAE;4BACrD,KAAK;gCAACD,OAAOE,QAAQ,CAAC,EAAE;gCAAGF,OAAOE,QAAQ,CAAC,EAAE;6BAAE;wBACjD;oBACF;gBACA,KAAK;oBAAc;wBACjB,MAAMH,eAAeN,gBAAgB,KAAK,CAAC;wBAC3C,IAAI,CAACM,cACH,MAAM,IAAIV,MACR,CAAC,2DAA2D,EAAEI,gBAAgB,CAAC,CAAC;wBAEpF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACR,SAAS;gCAACE,OAAOD,YAAY,CAAC,EAAE;gCAAGC,OAAOD,YAAY,CAAC,EAAE;6BAAE;wBAC7D;oBACF;gBACA,KAAK;oBAAU;wBACb,MAAMI,MAAMlB,kBAAkBQ,iBAAiB;wBAC/C,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRK;wBACF;oBACF;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGL,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBACH,OAAO;wBAAE,GAAGA,UAAU;wBAAE,QAAQ;oBAAO;gBAEzC,KAAK;oBAAQ;wBACX,MAAMM,gBAAgBX,gBAAgB,KAAK,CACzC;wBAEF,IAAI,CAACW,eACH,MAAM,IAAIf,MACR,CAAC,0CAA0C,EAAEI,gBAAgB,CAAC,CAAC;wBAGnE,MAAMY,UAAUL,OAAO,QAAQ,CAACI,aAAa,CAAC,EAAE,EAAE;wBAClD,MAAME,aAAaD,AAAU,OAAVA;wBACnB,OAAO;4BACL,GAAGP,UAAU;4BACb,QAAQ;4BACRQ;wBACF;oBACF;gBACA,KAAK;oBACH,OAAO;wBAAE,GAAGR,UAAU;wBAAE,QAAQ;oBAAW;gBAE7C,KAAK;oBAAY;wBACf,MAAMS,cAActB,kBAClBQ,iBACA;wBAEF,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRS;wBACF;oBACF;gBACA,KAAK;oBAAa;wBAChB,MAAMZ,UAAUV,kBAAkBQ,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA,KAAK;oBAAQ;wBACX,MAAMA,UAAUV,kBAAkBQ,iBAAiB;wBACnD,OAAO;4BACL,GAAGK,UAAU;4BACb,QAAQ;4BACRH;wBACF;oBACF;gBACA;oBACE,MAAM,IAAIN,MACR,CAAC,qBAAqB,EAAEQ,WAAW,OAAO,EAAEJ,gBAAgB,CAAC,CAAC;YAEpE;QACF;QACA,MAAM,IAAIJ,MAAM,CAAC,wBAAwB,EAAEI,iBAAiB;IAC9D,EAAE,OAAOe,OAAO;QACd,MAAMC,eAAeD,iBAAiBnB,QAAQmB,MAAM,OAAO,GAAGE,OAAOF;QACrE,MAAM,IAAInB,MACR,CAAC,wBAAwB,EAAEoB,aAAa,OAAO,EAAEhB,gBAAgB,CAAC,CAAC;IAEvE;AACF;AAEO,SAASkB,qBAAqBC,OAAe;IAIlD,IAAIC;IAEJ,IAAID,QAAQ,QAAQ,CAAC,oBAAoB;QACvC,MAAME,QAAQF,QAAQ,KAAK,CAAC;QAC5B,MAAMG,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,eAAe,EAAEF,KAAK,CAAC,EAAE,EAAE;QAClDD,iBAAiB;YAAEE;YAAO,SAASC;QAAc;IACnD,OAAO,IAAIJ,QAAQ,QAAQ,CAAC,eAAe;QACzC,MAAME,QAAQF,QAAQ,KAAK,CAAC;QAC5B,MAAMG,QAAQD,KAAK,CAAC,EAAE,CAAC,IAAI;QAC3B,MAAME,gBAAgB,CAAC,UAAU,EAAEF,KAAK,CAAC,EAAE,EAAE;QAC7CD,iBAAiB;YAAEE;YAAO,SAASC;QAAc;IACnD,OAAO,IAAIJ,QAAQ,QAAQ,CAAC,aAAa;QACvC,MAAME,QAAQF,QAAQ,KAAK,CAAC;QAC5B,MAAMG,QAAQD,KAAK,CAAC,EAAE,CACnB,OAAO,CAAC,YAAY,IACpB,OAAO,CAAC,cAAc,IACtB,IAAI;QACP,MAAME,gBAAgBF,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,IAAI;QAC9DD,iBAAiB;YAAEE;YAAO,SAASC;QAAc;IACnD,OACEH,iBAAiB;QAAE,OAAO;QAAID;IAAQ;IAGxC7B,MAAM,wBAAwB6B;IAC9B7B,MAAM,yBAAyB8B,eAAe,KAAK;IACnD9B,MAAM,uBAAuB8B,eAAe,OAAO;IACnD,OAAOA;AACT;AAEO,SAASI,6BAA6BL,OAAe;IAI1D,MAAMpB,WAAWmB,qBAAqBC;IACtC,MAAMM,SAAS3B,2BAA2BC;IAC1C,OAAO;QAAEA;QAAU0B;IAAO;AAC5B"}

package/dist/es/ai-model/models/kimi.mjs CHANGED Viewed

@@ -12,7 +12,7 @@ const kimiPixelPointCoordinatesMeta = {
 };
 function parseKimiRawLocateValue(input) {
     const point = parseCoordinateList(input, 'point');
-    if (point.length < 2) throw new Error(`invalid point data: ${JSON.stringify(input)} `);
+    if (2 !== point.length) throw new Error(`invalid point data: ${JSON.stringify(input)} `);
     const [x, y] = point;
     const coordinatesMeta = x <= 1 && y <= 1 ? kimiNormalizedPointCoordinatesMeta : kimiPixelPointCoordinatesMeta;
     return createLocateResultValue(coordinatesMeta, [

package/dist/es/ai-model/models/kimi.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/models/kimi.mjs","sources":["../../../../src/ai-model/models/kimi.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n ChatCompletionCallContext,\n ChatCompletionParamsResult,\n ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n type LocateResultValue,\n createLocateResultValue,\n parseCoordinateList,\n} from '../shared/model-locate-result';\n\nconst kimiNormalizedPointCoordinatesMeta = {\n shape: 'point',\n order: 'xy',\n normalizedBy: 1,\n rounding: 'round',\n} as const;\nconst kimiPixelPointCoordinatesMeta = {\n shape: 'point',\n order: 'xy',\n rounding: 'round',\n} as const;\n\nfunction parseKimiRawLocateValue(input: unknown): LocateResultValue {\n const point = parseCoordinateList(input, 'point');\n if (point.length < 2) {\n throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n }\n const [x, y] = point;\n // Keep this compatible with OSWorld's Kimi adapter: values <= 1 are\n // normalized coordinates, otherwise they are treated as screenshot pixels.\n const coordinatesMeta =\n x <= 1 && y <= 1\n ? kimiNormalizedPointCoordinatesMeta\n : kimiPixelPointCoordinatesMeta;\n return createLocateResultValue(coordinatesMeta, [x, y]);\n}\n\nconst buildKimiChatCompletionParams = (\n input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n const { midsceneDefaults, userConfig } = input;\n const { reasoningEnabled } = userConfig;\n const effectiveReasoningEnabled = reasoningEnabled ?? false;\n const commonOverrideConfig: Record<string, unknown> = {};\n\n // kimi disallow custom temperature\n commonOverrideConfig.temperature = undefined;\n\n // Kimi Chat Completions response_format:\n // https://platform.kimi.com/docs/api/chat\n if (\n userConfig.responseFormat !== 'none' &&\n input.expectedJsonObjectResponse\n ) {\n commonOverrideConfig.response_format = { type: 'json_object' };\n }\n\n const modelSpecificConfig: Record<string, unknown> = {\n thinking: {\n type: effectiveReasoningEnabled ? 'enabled' : 'disabled',\n },\n };\n\n return {\n config: {\n ...midsceneDefaults,\n ...commonOverrideConfig,\n ...modelSpecificConfig,\n },\n };\n};\n\nconst buildKimi3ChatCompletionParams = (\n input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n const { midsceneDefaults, userConfig } = input;\n const commonOverrideConfig: Record<string, unknown> = {};\n\n // Kimi disallows custom temperature.\n commonOverrideConfig.temperature = undefined;\n\n if (\n userConfig.responseFormat !== 'none' &&\n input.expectedJsonObjectResponse\n ) {\n commonOverrideConfig.response_format = { type: 'json_object' };\n }\n\n // Kimi K3 currently only supports reasoning_effort=\"max\"; its docs say\n // additional effort levels will be available later.\n const reasoningEffort = userConfig.reasoningEffort;\n\n return {\n config: {\n ...midsceneDefaults,\n ...commonOverrideConfig,\n ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}),\n },\n };\n};\n\nexport const kimiAdapters = {\n kimi: {\n chatCompletion: {\n unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n buildChatCompletionParams: buildKimiChatCompletionParams,\n useReasoningAsContentFallback: true,\n },\n locate: {\n element: {\n resultFormat: {\n coordinates: kimiNormalizedPointCoordinatesMeta,\n parseRawLocateValue: parseKimiRawLocateValue,\n },\n },\n },\n },\n kimi3: {\n chatCompletion: {\n unsupportedUserConfig: ['reasoningEnabled', 'reasoningBudget'],\n buildChatCompletionParams: buildKimi3ChatCompletionParams,\n useReasoningAsContentFallback: true,\n // Kimi K3 multi-turn and tool calls must replay the complete assistant\n // message, including reasoning_content and tool_calls.\n // https://platform.kimi.com/docs/guide/use-thinking-effort\n replayRawAssistantMessage: true,\n },\n locate: {\n element: {\n resultFormat: {\n coordinates: kimiNormalizedPointCoordinatesMeta,\n parseRawLocateValue: parseKimiRawLocateValue,\n },\n },\n },\n },\n} satisfies Pick<\n Record<TModelFamily, ModelAdapterDefinition>,\n 'kimi' \| 'kimi3'\n>;\n"],"names":["kimiNormalizedPointCoordinatesMeta","kimiPixelPointCoordinatesMeta","parseKimiRawLocateValue","input","point","parseCoordinateList","Error","JSON","x","y","coordinatesMeta","createLocateResultValue","buildKimiChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","effectiveReasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","buildKimi3ChatCompletionParams","reasoningEffort","kimiAdapters"],"mappings":";AAYA,MAAMA,qCAAqC;IACzC,OAAO;IACP,OAAO;IACP,cAAc;IACd,UAAU;AACZ;AACA,MAAMC,gCAAgC;IACpC,OAAO;IACP,OAAO;IACP,UAAU;AACZ;AAEA,SAASC,wBAAwBC,KAAc;IAC7C,MAAMC,QAAQC,oBAAoBF,OAAO;IACzC,IAAIC,MAAM,MAAM,~~GAAG~~,~~GACjB,~~MAAM,IAAIE,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACJ,OAAO,CAAC,CAAC;IAEjE,MAAM,CAACK,GAAGC,EAAE,GAAGL;IAGf,MAAMM,kBACJF,KAAK,KAAKC,KAAK,IACXT,qCACAC;IACN,OAAOU,wBAAwBD,iBAAiB;QAACF;QAAGC;KAAE;AACxD;AAEA,MAAMG,gCAAgC,CACpCT;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAM,EAAEY,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,4BAA4BD,oBAAoB;IACtD,MAAME,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAInC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,sBAA+C;QACnD,UAAU;YACR,MAAMH,4BAA4B,YAAY;QAChD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,iCAAiC,CACrCjB;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAMc,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAEnC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAK/D,MAAMI,kBAAkBP,WAAW,eAAe;IAElD,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAII,kBAAkB;gBAAE,kBAAkBA;YAAgB,IAAI,CAAC,CAAC;QAClE;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,MAAM;QACJ,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BV;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAaZ;oBACb,qBAAqBE;gBACvB;YACF;QACF;IACF;IACA,OAAO;QACL,gBAAgB;YACd,uBAAuB;gBAAC;gBAAoB;aAAkB;YAC9D,2BAA2BkB;YAC3B,+BAA+B;YAI/B,2BAA2B;QAC7B;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAapB;oBACb,qBAAqBE;gBACvB;YACF;QACF;IACF;AACF"}
1	+ {"version":3,"file":"ai-model/models/kimi.mjs","sources":["../../../../src/ai-model/models/kimi.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type {\n ChatCompletionCallContext,\n ChatCompletionParamsResult,\n ModelAdapterDefinition,\n} from '../model-adapter/types';\nimport {\n type LocateResultValue,\n createLocateResultValue,\n parseCoordinateList,\n} from '../shared/model-locate-result';\n\nconst kimiNormalizedPointCoordinatesMeta = {\n shape: 'point',\n order: 'xy',\n normalizedBy: 1,\n rounding: 'round',\n} as const;\nconst kimiPixelPointCoordinatesMeta = {\n shape: 'point',\n order: 'xy',\n rounding: 'round',\n} as const;\n\nfunction parseKimiRawLocateValue(input: unknown): LocateResultValue {\n const point = parseCoordinateList(input, 'point');\n if (point.length !== 2) {\n throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n }\n const [x, y] = point;\n // Keep this compatible with OSWorld's Kimi adapter: values <= 1 are\n // normalized coordinates, otherwise they are treated as screenshot pixels.\n const coordinatesMeta =\n x <= 1 && y <= 1\n ? kimiNormalizedPointCoordinatesMeta\n : kimiPixelPointCoordinatesMeta;\n return createLocateResultValue(coordinatesMeta, [x, y]);\n}\n\nconst buildKimiChatCompletionParams = (\n input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n const { midsceneDefaults, userConfig } = input;\n const { reasoningEnabled } = userConfig;\n const effectiveReasoningEnabled = reasoningEnabled ?? false;\n const commonOverrideConfig: Record<string, unknown> = {};\n\n // kimi disallow custom temperature\n commonOverrideConfig.temperature = undefined;\n\n // Kimi Chat Completions response_format:\n // https://platform.kimi.com/docs/api/chat\n if (\n userConfig.responseFormat !== 'none' &&\n input.expectedJsonObjectResponse\n ) {\n commonOverrideConfig.response_format = { type: 'json_object' };\n }\n\n const modelSpecificConfig: Record<string, unknown> = {\n thinking: {\n type: effectiveReasoningEnabled ? 'enabled' : 'disabled',\n },\n };\n\n return {\n config: {\n ...midsceneDefaults,\n ...commonOverrideConfig,\n ...modelSpecificConfig,\n },\n };\n};\n\nconst buildKimi3ChatCompletionParams = (\n input: ChatCompletionCallContext,\n): ChatCompletionParamsResult => {\n const { midsceneDefaults, userConfig } = input;\n const commonOverrideConfig: Record<string, unknown> = {};\n\n // Kimi disallows custom temperature.\n commonOverrideConfig.temperature = undefined;\n\n if (\n userConfig.responseFormat !== 'none' &&\n input.expectedJsonObjectResponse\n ) {\n commonOverrideConfig.response_format = { type: 'json_object' };\n }\n\n // Kimi K3 currently only supports reasoning_effort=\"max\"; its docs say\n // additional effort levels will be available later.\n const reasoningEffort = userConfig.reasoningEffort;\n\n return {\n config: {\n ...midsceneDefaults,\n ...commonOverrideConfig,\n ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}),\n },\n };\n};\n\nexport const kimiAdapters = {\n kimi: {\n chatCompletion: {\n unsupportedUserConfig: ['reasoningEffort', 'reasoningBudget'],\n buildChatCompletionParams: buildKimiChatCompletionParams,\n useReasoningAsContentFallback: true,\n },\n locate: {\n element: {\n resultFormat: {\n coordinates: kimiNormalizedPointCoordinatesMeta,\n parseRawLocateValue: parseKimiRawLocateValue,\n },\n },\n },\n },\n kimi3: {\n chatCompletion: {\n unsupportedUserConfig: ['reasoningEnabled', 'reasoningBudget'],\n buildChatCompletionParams: buildKimi3ChatCompletionParams,\n useReasoningAsContentFallback: true,\n // Kimi K3 multi-turn and tool calls must replay the complete assistant\n // message, including reasoning_content and tool_calls.\n // https://platform.kimi.com/docs/guide/use-thinking-effort\n replayRawAssistantMessage: true,\n },\n locate: {\n element: {\n resultFormat: {\n coordinates: kimiNormalizedPointCoordinatesMeta,\n parseRawLocateValue: parseKimiRawLocateValue,\n },\n },\n },\n },\n} satisfies Pick<\n Record<TModelFamily, ModelAdapterDefinition>,\n 'kimi' \| 'kimi3'\n>;\n"],"names":["kimiNormalizedPointCoordinatesMeta","kimiPixelPointCoordinatesMeta","parseKimiRawLocateValue","input","point","parseCoordinateList","Error","JSON","x","y","coordinatesMeta","createLocateResultValue","buildKimiChatCompletionParams","midsceneDefaults","userConfig","reasoningEnabled","effectiveReasoningEnabled","commonOverrideConfig","undefined","modelSpecificConfig","buildKimi3ChatCompletionParams","reasoningEffort","kimiAdapters"],"mappings":";AAYA,MAAMA,qCAAqC;IACzC,OAAO;IACP,OAAO;IACP,cAAc;IACd,UAAU;AACZ;AACA,MAAMC,gCAAgC;IACpC,OAAO;IACP,OAAO;IACP,UAAU;AACZ;AAEA,SAASC,wBAAwBC,KAAc;IAC7C,MAAMC,QAAQC,oBAAoBF,OAAO;IACzC,IAAIC,AAAiB,MAAjBA,MAAM,MAAM,EACd,MAAM,IAAIE,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACJ,OAAO,CAAC,CAAC;IAEjE,MAAM,CAACK,GAAGC,EAAE,GAAGL;IAGf,MAAMM,kBACJF,KAAK,KAAKC,KAAK,IACXT,qCACAC;IACN,OAAOU,wBAAwBD,iBAAiB;QAACF;QAAGC;KAAE;AACxD;AAEA,MAAMG,gCAAgC,CACpCT;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAM,EAAEY,gBAAgB,EAAE,GAAGD;IAC7B,MAAME,4BAA4BD,oBAAoB;IACtD,MAAME,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAInC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAG/D,MAAME,sBAA+C;QACnD,UAAU;YACR,MAAMH,4BAA4B,YAAY;QAChD;IACF;IAEA,OAAO;QACL,QAAQ;YACN,GAAGH,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAGE,mBAAmB;QACxB;IACF;AACF;AAEA,MAAMC,iCAAiC,CACrCjB;IAEA,MAAM,EAAEU,gBAAgB,EAAEC,UAAU,EAAE,GAAGX;IACzC,MAAMc,uBAAgD,CAAC;IAGvDA,qBAAqB,WAAW,GAAGC;IAEnC,IACEJ,AAA8B,WAA9BA,WAAW,cAAc,IACzBX,MAAM,0BAA0B,EAEhCc,qBAAqB,eAAe,GAAG;QAAE,MAAM;IAAc;IAK/D,MAAMI,kBAAkBP,WAAW,eAAe;IAElD,OAAO;QACL,QAAQ;YACN,GAAGD,gBAAgB;YACnB,GAAGI,oBAAoB;YACvB,GAAII,kBAAkB;gBAAE,kBAAkBA;YAAgB,IAAI,CAAC,CAAC;QAClE;IACF;AACF;AAEO,MAAMC,eAAe;IAC1B,MAAM;QACJ,gBAAgB;YACd,uBAAuB;gBAAC;gBAAmB;aAAkB;YAC7D,2BAA2BV;YAC3B,+BAA+B;QACjC;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAaZ;oBACb,qBAAqBE;gBACvB;YACF;QACF;IACF;IACA,OAAO;QACL,gBAAgB;YACd,uBAAuB;gBAAC;gBAAoB;aAAkB;YAC9D,2BAA2BkB;YAC3B,+BAA+B;YAI/B,2BAA2B;QAC7B;QACA,QAAQ;YACN,SAAS;gBACP,cAAc;oBACZ,aAAapB;oBACb,qBAAqBE;gBACvB;YACF;QACF;IACF;AACF"}

package/dist/es/ai-model/service-caller/call-ai.mjs CHANGED Viewed

@@ -126,7 +126,16 @@ async function callModelWithRetry(input) {
             });
             if (attempt === maxAttempts) throw appendAIRequestFailureSummary(lastError, attemptErrors, maxAttempts);
             getDebug('ai:call', {
-                console: true
+                console: true,
+                warning: {
+                    kind: 'model-request-retry',
+                    modelName: config.modelName,
+                    error: lastError,
+                    callId: input.internalCallId,
+                    attempt,
+                    maxAttempts,
+                    delayMs: retryInterval
+                }
             })(`AI call failed (attempt ${attempt}/${maxAttempts}), retrying in ${retryInterval}ms... Error: ${lastError.message}`);
         }
         await waitForRetry(retryInterval, options?.abortSignal);

package/dist/es/ai-model/service-caller/call-ai.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/service-caller/call-ai.mjs","sources":["../../../../src/ai-model/service-caller/call-ai.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert, uuid } from '@midscene/shared/utils';\nimport type { ModelRuntime } from '../models';\nimport { callCodex, prepareCodexCall } from './codex/call-codex';\nimport { isCodexAppServerProvider } from './codex/codex-app-server';\nimport {\n isModelCallRecordingEnabled,\n recordModelCallEvent,\n} from './model-call-recorder';\nimport { type PreparedOpenAIInput, callOpenAI } from './openai/call-openai';\nimport { prepareChatCompletion } from './openai/chat-completion/chat-completion';\nimport { prepareResponses } from './openai/responses/responses';\nimport {\n buildRequestAbortSignal,\n resolveEffectiveTimeoutMs,\n runWithAbortSignal,\n waitForRetry,\n} from './request-timeout';\nimport { callSystemOne, prepareSystemOne } from './system-one';\nimport type { ModelCallMessages } from './types';\nimport type {\n AICallResult,\n CallAIOptions,\n ModelCallContext,\n ModelCallResult,\n} from './types';\nimport {\n appendAIRequestFailureSummary,\n buildUsageInfo,\n nextInternalCallId,\n normalizeRetryCount,\n toError,\n} from './utils';\n\nconst DEFAULT_MODEL_PROTOCOL = 'openai-chat';\n\nexport async function callAI(\n messages: ModelCallMessages,\n modelRuntime: ModelRuntime,\n options?: CallAIOptions,\n): Promise<AICallResult> {\n options?.abortSignal?.throwIfAborted();\n const isStreaming = options?.stream === true;\n if (isStreaming) {\n // Explicit streaming is currently used only by recorder code generation.\n // Streaming can also be aggregated without onChunk, but then the caller\n // cannot consume incremental output, which is the purpose of this option.\n // Environment-enabled streaming supports that aggregation-only use case.\n // Once onChunk delivers a chunk, later errors are not retried to avoid\n // delivering duplicate content. Errors before the first chunk can retry.\n assert(\n typeof options?.onChunk === 'function',\n 'onChunk is required when stream: true is explicitly specified',\n );\n }\n\n // Low-level callers without a TaskRunner still need a stable ID for the\n // lifetime of this model call (including its network retries).\n const executionId = modelRuntime.executionId ?? `unscoped-${uuid()}`;\n\n // Stable internal ID for this call, used by the agent to deduplicate usage\n // across the onUsage callback and the task-dump-based collectUsageMetrics()\n // path when the provider does not return a request_id.\n const internalCallId = nextInternalCallId();\n\n const { config: modelConfig } = modelRuntime;\n\n const prepare = async (): Promise<PreparedModelInput> => {\n if (options?.systemOneRequest) {\n assert(\n modelConfig.modelFamily === 'jev',\n 'System One requests require the Jev model family',\n );\n return {\n protocol: 'system-one',\n input: await prepareSystemOne(modelConfig, options.systemOneRequest),\n };\n }\n const input = { messages, modelRuntime, options };\n if (isCodexAppServerProvider(modelConfig.openaiBaseURL)) {\n return { protocol: 'codex', input: prepareCodexCall(input) };\n }\n const protocol = modelConfig.protocol ?? DEFAULT_MODEL_PROTOCOL;\n\n assert(\n modelRuntime.adapter.supportedProtocols.includes(protocol),\n `Model adapter \"${modelConfig.modelFamily ?? 'default'}\" does not support protocol \"${protocol}\"`,\n );\n if (protocol === 'openai-responses') {\n return { protocol: 'responses', input: await prepareResponses(input) };\n }\n return {\n protocol: 'chat-completion',\n input: await prepareChatCompletion(input),\n };\n };\n\n const prepared = options?.abortSignal\n ? await runWithAbortSignal(options.abortSignal, prepare)\n : await prepare();\n\n const result = await callModelWithRetry({\n messages,\n modelRuntime,\n options,\n executionId,\n internalCallId,\n prepared,\n });\n\n const {\n rawUsage,\n timeCost,\n totalTimeCost,\n retryCount,\n requestId,\n responseModelName,\n ...response\n } = result;\n\n const rawUsageInfo = buildUsageInfo({\n apiProtocol: prepared.protocol,\n stream: response.isStreamed,\n usageData: rawUsage,\n timeCost,\n totalTimeCost,\n retryCount,\n requestId,\n responseModelName,\n modelName: modelConfig.modelName,\n modelDescription: modelConfig.modelDescription,\n slot: modelConfig.slot,\n internalCallId,\n });\n\n const usage = rawUsageInfo\n ? { ...rawUsageInfo, modelRouting: modelRuntime.modelRouting }\n : undefined;\n\n const debugProfileStats = getDebug('ai:profile:stats');\n const debugProfileDetail = getDebug('ai:profile:detail');\n debugProfileStats(\n `model, ${modelConfig.modelName}, mode, ${modelConfig.modelFamily \|\| 'default'}, streaming, ${response.isStreamed}, prompt-tokens, ${usage?.prompt_tokens ?? ''}, completion-tokens, ${usage?.completion_tokens ?? ''}, total-tokens, ${usage?.total_tokens ?? ''}, cached-input, ${usage?.cached_input ?? ''}, cost-ms, ${timeCost ?? ''}, requestId, ${requestId ?? ''}, slot, ${modelConfig.slot}, configured-temperature, ${modelConfig.temperature ?? ''}`,\n );\n debugProfileDetail(`model usage detail: ${JSON.stringify(usage)}`);\n\n if (usage && modelRuntime.onUsage) {\n modelRuntime.onUsage(usage);\n }\n return { ...response, usage };\n}\n\ntype PreparedModelInput =\n \| {\n protocol: 'system-one';\n input: Awaited<ReturnType<typeof prepareSystemOne>>;\n }\n \| { protocol: 'codex'; input: ReturnType<typeof prepareCodexCall> }\n \| PreparedOpenAIInput;\n\ntype ModelCallInput = {\n messages: ModelCallMessages;\n modelRuntime: ModelRuntime;\n options?: CallAIOptions;\n executionId: string;\n internalCallId: string;\n prepared: PreparedModelInput;\n};\n\n/** Carries the original failure across the retry boundary without replaying output. */\nclass NonRetryableModelCallError extends Error {\n readonly cause: Error;\n\n constructor(cause: Error) {\n super(cause.message, { cause });\n this.cause = cause;\n }\n}\n\nasync function callModelWithRetry(\n input: ModelCallInput,\n): Promise<ModelCallResult & { totalTimeCost: number; retryCount: number }> {\n const startTime = Date.now();\n const {\n modelRuntime: { config },\n options,\n } = input;\n const maxAttempts = normalizeRetryCount(config.retryCount) + 1;\n const retryInterval = config.retryInterval ?? 2000;\n const attemptErrors: Array<{ attempt: number; error: unknown }> = [];\n\n for (let attempt = 1; ; attempt++) {\n options?.abortSignal?.throwIfAborted();\n try {\n const result = await callModelOnce(input, attempt);\n return {\n ...result,\n totalTimeCost: Date.now() - startTime,\n retryCount: attempt - 1,\n };\n } catch (error) {\n options?.abortSignal?.throwIfAborted();\n if (error instanceof NonRetryableModelCallError) {\n throw error.cause;\n }\n const lastError = toError(error);\n attemptErrors.push({ attempt, error: lastError });\n if (attempt === maxAttempts) {\n throw appendAIRequestFailureSummary(\n lastError,\n attemptErrors,\n maxAttempts,\n );\n }\n getDebug('ai:call', { console: true })(\n `AI call failed (attempt ${attempt}/${maxAttempts}), retrying in ${retryInterval}ms... Error: ${lastError.message}`,\n );\n }\n await waitForRetry(retryInterval, options?.abortSignal);\n }\n}\n\nasync function callModelOnce(\n {\n messages,\n modelRuntime,\n options,\n executionId,\n internalCallId,\n prepared,\n }: ModelCallInput,\n attempt: number,\n): Promise<ModelCallResult> {\n const { config: modelConfig } = modelRuntime;\n const recordingEnabled = isModelCallRecordingEnabled();\n const recordEvent =\n recordingEnabled \|\| modelRuntime.onRequest\n ? (event: Record<string, unknown>) => {\n if (event.type === 'request') {\n const request = event.request as { url?: string } \| undefined;\n modelRuntime.onRequest?.(request?.url);\n }\n if (!recordingEnabled) return;\n void recordModelCallEvent({\n executionId,\n callId: internalCallId,\n semanticRetryAttempt: options?.semanticRetryAttempt,\n slot: modelConfig.slot,\n intent: modelConfig.intent,\n modelFamily: modelConfig.modelFamily,\n modelRouting: modelRuntime.modelRouting,\n ...event,\n attempt,\n });\n }\n : undefined;\n const effectiveTimeoutMs = resolveEffectiveTimeoutMs(modelConfig.timeout);\n const { signal: requestSignal, cleanup } = buildRequestAbortSignal(\n effectiveTimeoutMs,\n options?.abortSignal,\n );\n let deliveredChunk = false;\n const onChunk = options?.onChunk;\n const context: ModelCallContext = {\n messages,\n modelRuntime,\n executionId,\n recordEvent,\n requestSignal,\n effectiveTimeoutMs,\n options: onChunk\n ? {\n ...options,\n onChunk: (chunk) => {\n requestSignal.throwIfAborted();\n deliveredChunk = true;\n onChunk(chunk);\n },\n }\n : options,\n };\n try {\n return await runWithAbortSignal(requestSignal, () =>\n prepared.protocol === 'system-one'\n ? callSystemOne(context, prepared.input)\n : prepared.protocol === 'codex'\n ? callCodex(context, prepared.input)\n : callOpenAI(context, prepared),\n );\n } catch (error) {\n options?.abortSignal?.throwIfAborted();\n const lastError = toError(error);\n if (deliveredChunk) {\n throw new NonRetryableModelCallError(lastError);\n }\n throw lastError;\n } finally {\n cleanup();\n }\n}\n"],"names":["DEFAULT_MODEL_PROTOCOL","callAI","messages","modelRuntime","options","isStreaming","assert","executionId","uuid","internalCallId","nextInternalCallId","modelConfig","prepare","prepareSystemOne","input","isCodexAppServerProvider","prepareCodexCall","protocol","prepareResponses","prepareChatCompletion","prepared","runWithAbortSignal","result","callModelWithRetry","rawUsage","timeCost","totalTimeCost","retryCount","requestId","responseModelName","response","rawUsageInfo","buildUsageInfo","usage","undefined","debugProfileStats","getDebug","debugProfileDetail","JSON","NonRetryableModelCallError","Error","cause","startTime","Date","config","maxAttempts","normalizeRetryCount","retryInterval","attemptErrors","attempt","callModelOnce","error","lastError","toError","appendAIRequestFailureSummary","waitForRetry","recordingEnabled","isModelCallRecordingEnabled","recordEvent","event","request","recordModelCallEvent","effectiveTimeoutMs","resolveEffectiveTimeoutMs","requestSignal","cleanup","buildRequestAbortSignal","deliveredChunk","onChunk","context","chunk","callSystemOne","callCodex","callOpenAI"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAkCA,MAAMA,yBAAyB;AAExB,eAAeC,OACpBC,QAA2B,EAC3BC,YAA0B,EAC1BC,OAAuB;IAEvBA,SAAS,aAAa;IACtB,MAAMC,cAAcD,SAAS,WAAW;IACxC,IAAIC,aAOFC,OACE,AAA4B,cAA5B,OAAOF,SAAS,SAChB;IAMJ,MAAMG,cAAcJ,aAAa,WAAW,IAAI,CAAC,SAAS,EAAEK,QAAQ;IAKpE,MAAMC,iBAAiBC;IAEvB,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGR;IAEhC,MAAMS,UAAU;QACd,IAAIR,SAAS,kBAAkB;YAC7BE,OACEK,AAA4B,UAA5BA,YAAY,WAAW,EACvB;YAEF,OAAO;gBACL,UAAU;gBACV,OAAO,MAAME,iBAAiBF,aAAaP,QAAQ,gBAAgB;YACrE;QACF;QACA,MAAMU,QAAQ;YAAEZ;YAAUC;YAAcC;QAAQ;QAChD,IAAIW,yBAAyBJ,YAAY,aAAa,GACpD,OAAO;YAAE,UAAU;YAAS,OAAOK,iBAAiBF;QAAO;QAE7D,MAAMG,WAAWN,YAAY,QAAQ,IAAIX;QAEzCM,OACEH,aAAa,OAAO,CAAC,kBAAkB,CAAC,QAAQ,CAACc,WACjD,CAAC,eAAe,EAAEN,YAAY,WAAW,IAAI,UAAU,6BAA6B,EAAEM,SAAS,CAAC,CAAC;QAEnG,IAAIA,AAAa,uBAAbA,UACF,OAAO;YAAE,UAAU;YAAa,OAAO,MAAMC,iBAAiBJ;QAAO;QAEvE,OAAO;YACL,UAAU;YACV,OAAO,MAAMK,sBAAsBL;QACrC;IACF;IAEA,MAAMM,WAAWhB,SAAS,cACtB,MAAMiB,mBAAmBjB,QAAQ,WAAW,EAAEQ,WAC9C,MAAMA;IAEV,MAAMU,SAAS,MAAMC,mBAAmB;QACtCrB;QACAC;QACAC;QACAG;QACAE;QACAW;IACF;IAEA,MAAM,EACJI,QAAQ,EACRC,QAAQ,EACRC,aAAa,EACbC,UAAU,EACVC,SAAS,EACTC,iBAAiB,EACjB,GAAGC,UACJ,GAAGR;IAEJ,MAAMS,eAAeC,eAAe;QAClC,aAAaZ,SAAS,QAAQ;QAC9B,QAAQU,SAAS,UAAU;QAC3B,WAAWN;QACXC;QACAC;QACAC;QACAC;QACAC;QACA,WAAWlB,YAAY,SAAS;QAChC,kBAAkBA,YAAY,gBAAgB;QAC9C,MAAMA,YAAY,IAAI;QACtBF;IACF;IAEA,MAAMwB,QAAQF,eACV;QAAE,GAAGA,YAAY;QAAE,cAAc5B,aAAa,YAAY;IAAC,IAC3D+B;IAEJ,MAAMC,oBAAoBC,SAAS;IACnC,MAAMC,qBAAqBD,SAAS;IACpCD,kBACE,CAAC,OAAO,EAAExB,YAAY,SAAS,CAAC,QAAQ,EAAEA,YAAY,WAAW,IAAI,UAAU,aAAa,EAAEmB,SAAS,UAAU,CAAC,iBAAiB,EAAEG,OAAO,iBAAiB,GAAG,qBAAqB,EAAEA,OAAO,qBAAqB,GAAG,gBAAgB,EAAEA,OAAO,gBAAgB,GAAG,gBAAgB,EAAEA,OAAO,gBAAgB,GAAG,WAAW,EAAER,YAAY,GAAG,aAAa,EAAEG,aAAa,GAAG,QAAQ,EAAEjB,YAAY,IAAI,CAAC,0BAA0B,EAAEA,YAAY,WAAW,IAAI,IAAI;IAEjc0B,mBAAmB,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACL,QAAQ;IAEjE,IAAIA,SAAS9B,aAAa,OAAO,EAC/BA,aAAa,OAAO,CAAC8B;IAEvB,OAAO;QAAE,GAAGH,QAAQ;QAAEG;IAAM;AAC9B;AAoBA,MAAMM,mCAAmCC;IAGvC,YAAYC,KAAY,CAAE;QACxB,KAAK,CAACA,MAAM,OAAO,EAAE;YAAEA;QAAM,IAH/B,uBAAS,SAAT;QAIE,IAAI,CAAC,KAAK,GAAGA;IACf;AACF;AAEA,eAAelB,mBACbT,KAAqB;IAErB,MAAM4B,YAAYC,KAAK,GAAG;IAC1B,MAAM,EACJ,cAAc,EAAEC,MAAM,EAAE,EACxBxC,OAAO,EACR,GAAGU;IACJ,MAAM+B,cAAcC,oBAAoBF,OAAO,UAAU,IAAI;IAC7D,MAAMG,gBAAgBH,OAAO,aAAa,IAAI;IAC9C,MAAMI,gBAA4D,EAAE;IAEpE,IAAK,IAAIC,UAAU,IAAKA,UAAW;QACjC7C,SAAS,aAAa;QACtB,IAAI;YACF,MAAMkB,SAAS,MAAM4B,cAAcpC,OAAOmC;YAC1C,OAAO;gBACL,GAAG3B,MAAM;gBACT,eAAeqB,KAAK,GAAG,KAAKD;gBAC5B,YAAYO,UAAU;YACxB;QACF,EAAE,OAAOE,OAAO;YACd/C,SAAS,aAAa;YACtB,IAAI+C,iBAAiBZ,4BACnB,MAAMY,MAAM,KAAK;YAEnB,MAAMC,YAAYC,QAAQF;YAC1BH,cAAc,IAAI,CAAC;gBAAEC;gBAAS,OAAOG;YAAU;YAC/C,IAAIH,YAAYJ,aACd,MAAMS,8BACJF,WACAJ,eACAH;YAGJT,SAAS,WAAW;gBAAE,SAAS;YAAK,GAClC,CAAC,wBAAwB,EAAEa,QAAQ,CAAC,EAAEJ,YAAY,eAAe,EAAEE,cAAc,aAAa,EAAEK,UAAU,OAAO,EAAE;QAEvH;QACA,MAAMG,aAAaR,eAAe3C,SAAS;IAC7C;AACF;AAEA,eAAe8C,cACb,EACEhD,QAAQ,EACRC,YAAY,EACZC,OAAO,EACPG,WAAW,EACXE,cAAc,EACdW,QAAQ,EACO,EACjB6B,OAAe;IAEf,MAAM,EAAE,QAAQtC,WAAW,EAAE,GAAGR;IAChC,MAAMqD,mBAAmBC;IACzB,MAAMC,cACJF,oBAAoBrD,aAAa,SAAS,GACtC,CAACwD;QACC,IAAIA,AAAe,cAAfA,MAAM,IAAI,EAAgB;YAC5B,MAAMC,UAAUD,MAAM,OAAO;YAC7BxD,aAAa,SAAS,GAAGyD,SAAS;QACpC;QACA,IAAI,CAACJ,kBAAkB;QAClBK,qBAAqB;YACxBtD;YACA,QAAQE;YACR,sBAAsBL,SAAS;YAC/B,MAAMO,YAAY,IAAI;YACtB,QAAQA,YAAY,MAAM;YAC1B,aAAaA,YAAY,WAAW;YACpC,cAAcR,aAAa,YAAY;YACvC,GAAGwD,KAAK;YACRV;QACF;IACF,IACAf;IACN,MAAM4B,qBAAqBC,0BAA0BpD,YAAY,OAAO;IACxE,MAAM,EAAE,QAAQqD,aAAa,EAAEC,OAAO,EAAE,GAAGC,wBACzCJ,oBACA1D,SAAS;IAEX,IAAI+D,iBAAiB;IACrB,MAAMC,UAAUhE,SAAS;IACzB,MAAMiE,UAA4B;QAChCnE;QACAC;QACAI;QACAmD;QACAM;QACAF;QACA,SAASM,UACL;YACE,GAAGhE,OAAO;YACV,SAAS,CAACkE;gBACRN,cAAc,cAAc;gBAC5BG,iBAAiB;gBACjBC,QAAQE;YACV;QACF,IACAlE;IACN;IACA,IAAI;QACF,OAAO,MAAMiB,mBAAmB2C,eAAe,IAC7C5C,AAAsB,iBAAtBA,SAAS,QAAQ,GACbmD,cAAcF,SAASjD,SAAS,KAAK,IACrCA,AAAsB,YAAtBA,SAAS,QAAQ,GACfoD,UAAUH,SAASjD,SAAS,KAAK,IACjCqD,WAAWJ,SAASjD;IAE9B,EAAE,OAAO+B,OAAO;QACd/C,SAAS,aAAa;QACtB,MAAMgD,YAAYC,QAAQF;QAC1B,IAAIgB,gBACF,MAAM,IAAI5B,2BAA2Ba;QAEvC,MAAMA;IACR,SAAU;QACRa;IACF;AACF"}
1	+ {"version":3,"file":"ai-model/service-caller/call-ai.mjs","sources":["../../../../src/ai-model/service-caller/call-ai.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert, uuid } from '@midscene/shared/utils';\nimport type { ModelRuntime } from '../models';\nimport { callCodex, prepareCodexCall } from './codex/call-codex';\nimport { isCodexAppServerProvider } from './codex/codex-app-server';\nimport {\n isModelCallRecordingEnabled,\n recordModelCallEvent,\n} from './model-call-recorder';\nimport { type PreparedOpenAIInput, callOpenAI } from './openai/call-openai';\nimport { prepareChatCompletion } from './openai/chat-completion/chat-completion';\nimport { prepareResponses } from './openai/responses/responses';\nimport {\n buildRequestAbortSignal,\n resolveEffectiveTimeoutMs,\n runWithAbortSignal,\n waitForRetry,\n} from './request-timeout';\nimport { callSystemOne, prepareSystemOne } from './system-one';\nimport type { ModelCallMessages } from './types';\nimport type {\n AICallResult,\n CallAIOptions,\n ModelCallContext,\n ModelCallResult,\n} from './types';\nimport {\n appendAIRequestFailureSummary,\n buildUsageInfo,\n nextInternalCallId,\n normalizeRetryCount,\n toError,\n} from './utils';\n\nconst DEFAULT_MODEL_PROTOCOL = 'openai-chat';\n\nexport async function callAI(\n messages: ModelCallMessages,\n modelRuntime: ModelRuntime,\n options?: CallAIOptions,\n): Promise<AICallResult> {\n options?.abortSignal?.throwIfAborted();\n const isStreaming = options?.stream === true;\n if (isStreaming) {\n // Explicit streaming is currently used only by recorder code generation.\n // Streaming can also be aggregated without onChunk, but then the caller\n // cannot consume incremental output, which is the purpose of this option.\n // Environment-enabled streaming supports that aggregation-only use case.\n // Once onChunk delivers a chunk, later errors are not retried to avoid\n // delivering duplicate content. Errors before the first chunk can retry.\n assert(\n typeof options?.onChunk === 'function',\n 'onChunk is required when stream: true is explicitly specified',\n );\n }\n\n // Low-level callers without a TaskRunner still need a stable ID for the\n // lifetime of this model call (including its network retries).\n const executionId = modelRuntime.executionId ?? `unscoped-${uuid()}`;\n\n // Stable internal ID for this call, used by the agent to deduplicate usage\n // across the onUsage callback and the task-dump-based collectUsageMetrics()\n // path when the provider does not return a request_id.\n const internalCallId = nextInternalCallId();\n\n const { config: modelConfig } = modelRuntime;\n\n const prepare = async (): Promise<PreparedModelInput> => {\n if (options?.systemOneRequest) {\n assert(\n modelConfig.modelFamily === 'jev',\n 'System One requests require the Jev model family',\n );\n return {\n protocol: 'system-one',\n input: await prepareSystemOne(modelConfig, options.systemOneRequest),\n };\n }\n const input = { messages, modelRuntime, options };\n if (isCodexAppServerProvider(modelConfig.openaiBaseURL)) {\n return { protocol: 'codex', input: prepareCodexCall(input) };\n }\n const protocol = modelConfig.protocol ?? DEFAULT_MODEL_PROTOCOL;\n\n assert(\n modelRuntime.adapter.supportedProtocols.includes(protocol),\n `Model adapter \"${modelConfig.modelFamily ?? 'default'}\" does not support protocol \"${protocol}\"`,\n );\n if (protocol === 'openai-responses') {\n return { protocol: 'responses', input: await prepareResponses(input) };\n }\n return {\n protocol: 'chat-completion',\n input: await prepareChatCompletion(input),\n };\n };\n\n const prepared = options?.abortSignal\n ? await runWithAbortSignal(options.abortSignal, prepare)\n : await prepare();\n\n const result = await callModelWithRetry({\n messages,\n modelRuntime,\n options,\n executionId,\n internalCallId,\n prepared,\n });\n\n const {\n rawUsage,\n timeCost,\n totalTimeCost,\n retryCount,\n requestId,\n responseModelName,\n ...response\n } = result;\n\n const rawUsageInfo = buildUsageInfo({\n apiProtocol: prepared.protocol,\n stream: response.isStreamed,\n usageData: rawUsage,\n timeCost,\n totalTimeCost,\n retryCount,\n requestId,\n responseModelName,\n modelName: modelConfig.modelName,\n modelDescription: modelConfig.modelDescription,\n slot: modelConfig.slot,\n internalCallId,\n });\n\n const usage = rawUsageInfo\n ? { ...rawUsageInfo, modelRouting: modelRuntime.modelRouting }\n : undefined;\n\n const debugProfileStats = getDebug('ai:profile:stats');\n const debugProfileDetail = getDebug('ai:profile:detail');\n debugProfileStats(\n `model, ${modelConfig.modelName}, mode, ${modelConfig.modelFamily \|\| 'default'}, streaming, ${response.isStreamed}, prompt-tokens, ${usage?.prompt_tokens ?? ''}, completion-tokens, ${usage?.completion_tokens ?? ''}, total-tokens, ${usage?.total_tokens ?? ''}, cached-input, ${usage?.cached_input ?? ''}, cost-ms, ${timeCost ?? ''}, requestId, ${requestId ?? ''}, slot, ${modelConfig.slot}, configured-temperature, ${modelConfig.temperature ?? ''}`,\n );\n debugProfileDetail(`model usage detail: ${JSON.stringify(usage)}`);\n\n if (usage && modelRuntime.onUsage) {\n modelRuntime.onUsage(usage);\n }\n return { ...response, usage };\n}\n\ntype PreparedModelInput =\n \| {\n protocol: 'system-one';\n input: Awaited<ReturnType<typeof prepareSystemOne>>;\n }\n \| { protocol: 'codex'; input: ReturnType<typeof prepareCodexCall> }\n \| PreparedOpenAIInput;\n\ntype ModelCallInput = {\n messages: ModelCallMessages;\n modelRuntime: ModelRuntime;\n options?: CallAIOptions;\n executionId: string;\n internalCallId: string;\n prepared: PreparedModelInput;\n};\n\n/** Carries the original failure across the retry boundary without replaying output. */\nclass NonRetryableModelCallError extends Error {\n readonly cause: Error;\n\n constructor(cause: Error) {\n super(cause.message, { cause });\n this.cause = cause;\n }\n}\n\nasync function callModelWithRetry(\n input: ModelCallInput,\n): Promise<ModelCallResult & { totalTimeCost: number; retryCount: number }> {\n const startTime = Date.now();\n const {\n modelRuntime: { config },\n options,\n } = input;\n const maxAttempts = normalizeRetryCount(config.retryCount) + 1;\n const retryInterval = config.retryInterval ?? 2000;\n const attemptErrors: Array<{ attempt: number; error: unknown }> = [];\n\n for (let attempt = 1; ; attempt++) {\n options?.abortSignal?.throwIfAborted();\n try {\n const result = await callModelOnce(input, attempt);\n return {\n ...result,\n totalTimeCost: Date.now() - startTime,\n retryCount: attempt - 1,\n };\n } catch (error) {\n options?.abortSignal?.throwIfAborted();\n if (error instanceof NonRetryableModelCallError) {\n throw error.cause;\n }\n const lastError = toError(error);\n attemptErrors.push({ attempt, error: lastError });\n if (attempt === maxAttempts) {\n throw appendAIRequestFailureSummary(\n lastError,\n attemptErrors,\n maxAttempts,\n );\n }\n getDebug('ai:call', {\n console: true,\n warning: {\n kind: 'model-request-retry',\n modelName: config.modelName,\n error: lastError,\n callId: input.internalCallId,\n attempt,\n maxAttempts,\n delayMs: retryInterval,\n },\n })(\n `AI call failed (attempt ${attempt}/${maxAttempts}), retrying in ${retryInterval}ms... Error: ${lastError.message}`,\n );\n }\n await waitForRetry(retryInterval, options?.abortSignal);\n }\n}\n\nasync function callModelOnce(\n {\n messages,\n modelRuntime,\n options,\n executionId,\n internalCallId,\n prepared,\n }: ModelCallInput,\n attempt: number,\n): Promise<ModelCallResult> {\n const { config: modelConfig } = modelRuntime;\n const recordingEnabled = isModelCallRecordingEnabled();\n const recordEvent =\n recordingEnabled \|\| modelRuntime.onRequest\n ? (event: Record<string, unknown>) => {\n if (event.type === 'request') {\n const request = event.request as { url?: string } \| undefined;\n modelRuntime.onRequest?.(request?.url);\n }\n if (!recordingEnabled) return;\n void recordModelCallEvent({\n executionId,\n callId: internalCallId,\n semanticRetryAttempt: options?.semanticRetryAttempt,\n slot: modelConfig.slot,\n intent: modelConfig.intent,\n modelFamily: modelConfig.modelFamily,\n modelRouting: modelRuntime.modelRouting,\n ...event,\n attempt,\n });\n }\n : undefined;\n const effectiveTimeoutMs = resolveEffectiveTimeoutMs(modelConfig.timeout);\n const { signal: requestSignal, cleanup } = buildRequestAbortSignal(\n effectiveTimeoutMs,\n options?.abortSignal,\n );\n let deliveredChunk = false;\n const onChunk = options?.onChunk;\n const context: ModelCallContext = {\n messages,\n modelRuntime,\n executionId,\n recordEvent,\n requestSignal,\n effectiveTimeoutMs,\n options: onChunk\n ? {\n ...options,\n onChunk: (chunk) => {\n requestSignal.throwIfAborted();\n deliveredChunk = true;\n onChunk(chunk);\n },\n }\n : options,\n };\n try {\n return await runWithAbortSignal(requestSignal, () =>\n prepared.protocol === 'system-one'\n ? callSystemOne(context, prepared.input)\n : prepared.protocol === 'codex'\n ? callCodex(context, prepared.input)\n : callOpenAI(context, prepared),\n );\n } catch (error) {\n options?.abortSignal?.throwIfAborted();\n const lastError = toError(error);\n if (deliveredChunk) {\n throw new NonRetryableModelCallError(lastError);\n }\n throw lastError;\n } finally {\n cleanup();\n }\n}\n"],"names":["DEFAULT_MODEL_PROTOCOL","callAI","messages","modelRuntime","options","isStreaming","assert","executionId","uuid","internalCallId","nextInternalCallId","modelConfig","prepare","prepareSystemOne","input","isCodexAppServerProvider","prepareCodexCall","protocol","prepareResponses","prepareChatCompletion","prepared","runWithAbortSignal","result","callModelWithRetry","rawUsage","timeCost","totalTimeCost","retryCount","requestId","responseModelName","response","rawUsageInfo","buildUsageInfo","usage","undefined","debugProfileStats","getDebug","debugProfileDetail","JSON","NonRetryableModelCallError","Error","cause","startTime","Date","config","maxAttempts","normalizeRetryCount","retryInterval","attemptErrors","attempt","callModelOnce","error","lastError","toError","appendAIRequestFailureSummary","waitForRetry","recordingEnabled","isModelCallRecordingEnabled","recordEvent","event","request","recordModelCallEvent","effectiveTimeoutMs","resolveEffectiveTimeoutMs","requestSignal","cleanup","buildRequestAbortSignal","deliveredChunk","onChunk","context","chunk","callSystemOne","callCodex","callOpenAI"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAkCA,MAAMA,yBAAyB;AAExB,eAAeC,OACpBC,QAA2B,EAC3BC,YAA0B,EAC1BC,OAAuB;IAEvBA,SAAS,aAAa;IACtB,MAAMC,cAAcD,SAAS,WAAW;IACxC,IAAIC,aAOFC,OACE,AAA4B,cAA5B,OAAOF,SAAS,SAChB;IAMJ,MAAMG,cAAcJ,aAAa,WAAW,IAAI,CAAC,SAAS,EAAEK,QAAQ;IAKpE,MAAMC,iBAAiBC;IAEvB,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGR;IAEhC,MAAMS,UAAU;QACd,IAAIR,SAAS,kBAAkB;YAC7BE,OACEK,AAA4B,UAA5BA,YAAY,WAAW,EACvB;YAEF,OAAO;gBACL,UAAU;gBACV,OAAO,MAAME,iBAAiBF,aAAaP,QAAQ,gBAAgB;YACrE;QACF;QACA,MAAMU,QAAQ;YAAEZ;YAAUC;YAAcC;QAAQ;QAChD,IAAIW,yBAAyBJ,YAAY,aAAa,GACpD,OAAO;YAAE,UAAU;YAAS,OAAOK,iBAAiBF;QAAO;QAE7D,MAAMG,WAAWN,YAAY,QAAQ,IAAIX;QAEzCM,OACEH,aAAa,OAAO,CAAC,kBAAkB,CAAC,QAAQ,CAACc,WACjD,CAAC,eAAe,EAAEN,YAAY,WAAW,IAAI,UAAU,6BAA6B,EAAEM,SAAS,CAAC,CAAC;QAEnG,IAAIA,AAAa,uBAAbA,UACF,OAAO;YAAE,UAAU;YAAa,OAAO,MAAMC,iBAAiBJ;QAAO;QAEvE,OAAO;YACL,UAAU;YACV,OAAO,MAAMK,sBAAsBL;QACrC;IACF;IAEA,MAAMM,WAAWhB,SAAS,cACtB,MAAMiB,mBAAmBjB,QAAQ,WAAW,EAAEQ,WAC9C,MAAMA;IAEV,MAAMU,SAAS,MAAMC,mBAAmB;QACtCrB;QACAC;QACAC;QACAG;QACAE;QACAW;IACF;IAEA,MAAM,EACJI,QAAQ,EACRC,QAAQ,EACRC,aAAa,EACbC,UAAU,EACVC,SAAS,EACTC,iBAAiB,EACjB,GAAGC,UACJ,GAAGR;IAEJ,MAAMS,eAAeC,eAAe;QAClC,aAAaZ,SAAS,QAAQ;QAC9B,QAAQU,SAAS,UAAU;QAC3B,WAAWN;QACXC;QACAC;QACAC;QACAC;QACAC;QACA,WAAWlB,YAAY,SAAS;QAChC,kBAAkBA,YAAY,gBAAgB;QAC9C,MAAMA,YAAY,IAAI;QACtBF;IACF;IAEA,MAAMwB,QAAQF,eACV;QAAE,GAAGA,YAAY;QAAE,cAAc5B,aAAa,YAAY;IAAC,IAC3D+B;IAEJ,MAAMC,oBAAoBC,SAAS;IACnC,MAAMC,qBAAqBD,SAAS;IACpCD,kBACE,CAAC,OAAO,EAAExB,YAAY,SAAS,CAAC,QAAQ,EAAEA,YAAY,WAAW,IAAI,UAAU,aAAa,EAAEmB,SAAS,UAAU,CAAC,iBAAiB,EAAEG,OAAO,iBAAiB,GAAG,qBAAqB,EAAEA,OAAO,qBAAqB,GAAG,gBAAgB,EAAEA,OAAO,gBAAgB,GAAG,gBAAgB,EAAEA,OAAO,gBAAgB,GAAG,WAAW,EAAER,YAAY,GAAG,aAAa,EAAEG,aAAa,GAAG,QAAQ,EAAEjB,YAAY,IAAI,CAAC,0BAA0B,EAAEA,YAAY,WAAW,IAAI,IAAI;IAEjc0B,mBAAmB,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACL,QAAQ;IAEjE,IAAIA,SAAS9B,aAAa,OAAO,EAC/BA,aAAa,OAAO,CAAC8B;IAEvB,OAAO;QAAE,GAAGH,QAAQ;QAAEG;IAAM;AAC9B;AAoBA,MAAMM,mCAAmCC;IAGvC,YAAYC,KAAY,CAAE;QACxB,KAAK,CAACA,MAAM,OAAO,EAAE;YAAEA;QAAM,IAH/B,uBAAS,SAAT;QAIE,IAAI,CAAC,KAAK,GAAGA;IACf;AACF;AAEA,eAAelB,mBACbT,KAAqB;IAErB,MAAM4B,YAAYC,KAAK,GAAG;IAC1B,MAAM,EACJ,cAAc,EAAEC,MAAM,EAAE,EACxBxC,OAAO,EACR,GAAGU;IACJ,MAAM+B,cAAcC,oBAAoBF,OAAO,UAAU,IAAI;IAC7D,MAAMG,gBAAgBH,OAAO,aAAa,IAAI;IAC9C,MAAMI,gBAA4D,EAAE;IAEpE,IAAK,IAAIC,UAAU,IAAKA,UAAW;QACjC7C,SAAS,aAAa;QACtB,IAAI;YACF,MAAMkB,SAAS,MAAM4B,cAAcpC,OAAOmC;YAC1C,OAAO;gBACL,GAAG3B,MAAM;gBACT,eAAeqB,KAAK,GAAG,KAAKD;gBAC5B,YAAYO,UAAU;YACxB;QACF,EAAE,OAAOE,OAAO;YACd/C,SAAS,aAAa;YACtB,IAAI+C,iBAAiBZ,4BACnB,MAAMY,MAAM,KAAK;YAEnB,MAAMC,YAAYC,QAAQF;YAC1BH,cAAc,IAAI,CAAC;gBAAEC;gBAAS,OAAOG;YAAU;YAC/C,IAAIH,YAAYJ,aACd,MAAMS,8BACJF,WACAJ,eACAH;YAGJT,SAAS,WAAW;gBAClB,SAAS;gBACT,SAAS;oBACP,MAAM;oBACN,WAAWQ,OAAO,SAAS;oBAC3B,OAAOQ;oBACP,QAAQtC,MAAM,cAAc;oBAC5BmC;oBACAJ;oBACA,SAASE;gBACX;YACF,GACE,CAAC,wBAAwB,EAAEE,QAAQ,CAAC,EAAEJ,YAAY,eAAe,EAAEE,cAAc,aAAa,EAAEK,UAAU,OAAO,EAAE;QAEvH;QACA,MAAMG,aAAaR,eAAe3C,SAAS;IAC7C;AACF;AAEA,eAAe8C,cACb,EACEhD,QAAQ,EACRC,YAAY,EACZC,OAAO,EACPG,WAAW,EACXE,cAAc,EACdW,QAAQ,EACO,EACjB6B,OAAe;IAEf,MAAM,EAAE,QAAQtC,WAAW,EAAE,GAAGR;IAChC,MAAMqD,mBAAmBC;IACzB,MAAMC,cACJF,oBAAoBrD,aAAa,SAAS,GACtC,CAACwD;QACC,IAAIA,AAAe,cAAfA,MAAM,IAAI,EAAgB;YAC5B,MAAMC,UAAUD,MAAM,OAAO;YAC7BxD,aAAa,SAAS,GAAGyD,SAAS;QACpC;QACA,IAAI,CAACJ,kBAAkB;QAClBK,qBAAqB;YACxBtD;YACA,QAAQE;YACR,sBAAsBL,SAAS;YAC/B,MAAMO,YAAY,IAAI;YACtB,QAAQA,YAAY,MAAM;YAC1B,aAAaA,YAAY,WAAW;YACpC,cAAcR,aAAa,YAAY;YACvC,GAAGwD,KAAK;YACRV;QACF;IACF,IACAf;IACN,MAAM4B,qBAAqBC,0BAA0BpD,YAAY,OAAO;IACxE,MAAM,EAAE,QAAQqD,aAAa,EAAEC,OAAO,EAAE,GAAGC,wBACzCJ,oBACA1D,SAAS;IAEX,IAAI+D,iBAAiB;IACrB,MAAMC,UAAUhE,SAAS;IACzB,MAAMiE,UAA4B;QAChCnE;QACAC;QACAI;QACAmD;QACAM;QACAF;QACA,SAASM,UACL;YACE,GAAGhE,OAAO;YACV,SAAS,CAACkE;gBACRN,cAAc,cAAc;gBAC5BG,iBAAiB;gBACjBC,QAAQE;YACV;QACF,IACAlE;IACN;IACA,IAAI;QACF,OAAO,MAAMiB,mBAAmB2C,eAAe,IAC7C5C,AAAsB,iBAAtBA,SAAS,QAAQ,GACbmD,cAAcF,SAASjD,SAAS,KAAK,IACrCA,AAAsB,YAAtBA,SAAS,QAAQ,GACfoD,UAAUH,SAASjD,SAAS,KAAK,IACjCqD,WAAWJ,SAASjD;IAE9B,EAAE,OAAO+B,OAAO;QACd/C,SAAS,aAAa;QACtB,MAAMgD,YAAYC,QAAQF;QAC1B,IAAIgB,gBACF,MAAM,IAAI5B,2BAA2Ba;QAEvC,MAAMA;IACR,SAAU;QACRa;IACF;AACF"}

package/dist/es/ai-model/service-caller/openai/call-openai.mjs CHANGED Viewed

@@ -66,10 +66,15 @@ async function callOpenAI({ modelRuntime, options, executionId, recordEvent, req
             error: error instanceof Error ? error.message : String(error)
         });
         getDebug('ai:call', {
-            console: true
+            console: true,
+            warning: {
+                kind: 'model-request-detail',
+                modelName,
+                error
+            }
         })('call AI error', error);
         if (error instanceof AIResponseParseError) throw error;
-        throw new Error(`failed to call ${isStreaming ? 'streaming ' : ''}AI model service (${modelName}): ${error.message}${formatOpenAIAPIErrorDetails(error, openAIRequestContext)}\nTrouble shooting: https://midscenejs.com/model-provider.html`, {
+        throw new Error(`failed to call ${isStreaming ? 'streaming ' : ''}AI model service (${modelName}): ${error.message}${formatOpenAIAPIErrorDetails(error, openAIRequestContext)}\nTrouble shooting: https://midscenejs.com/model-config#verify-model-connectivity`, {
             cause: error
         });
     }

package/dist/es/ai-model/service-caller/openai/call-openai.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/service-caller/openai/call-openai.mjs","sources":["../../../../../src/ai-model/service-caller/openai/call-openai.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport type { ModelCallContext, ModelCallResult } from '../types';\nimport { AIResponseParseError } from '../utils';\nimport {\n callChatCompletion,\n type prepareChatCompletion,\n} from './chat-completion/chat-completion';\nimport { createClient } from './openai-client';\nimport { formatOpenAIAPIErrorDetails } from './openai-request-context';\nimport { callResponses, type prepareResponses } from './responses/responses';\n\nexport type PreparedOpenAIInput =\n \| {\n protocol: 'chat-completion';\n input: Awaited<ReturnType<typeof prepareChatCompletion>>;\n }\n \| {\n protocol: 'responses';\n input: Awaited<ReturnType<typeof prepareResponses>>;\n };\n\nexport async function callOpenAI(\n {\n modelRuntime,\n options,\n executionId,\n recordEvent,\n requestSignal,\n effectiveTimeoutMs,\n }: ModelCallContext,\n prepared: PreparedOpenAIInput,\n): Promise<ModelCallResult> {\n const { config: modelConfig } = modelRuntime;\n const { modelName } = modelConfig;\n const isStreaming = options?.stream ?? modelConfig.streamMode === 'stream';\n const { openai, openAIRequestContext } = await createClient({\n modelConfig,\n proxyAgent: prepared.input.proxyAgent,\n effectiveTimeoutMs,\n executionId,\n recordEvent,\n });\n const debugCall = getDebug('ai:call');\n debugCall(\n `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n );\n const requestOptions = {\n requestSignal,\n openAIRequestContext,\n onChunk: options?.onChunk ?? (() => undefined),\n recordEvent,\n };\n try {\n const startTime = Date.now();\n const response =\n prepared.protocol === 'responses'\n ? await callResponses(\n {\n ...requestOptions,\n responses: openai.responses,\n requestBodyParams: prepared.input.requestParams,\n },\n isStreaming,\n )\n : await callChatCompletion(\n {\n ...requestOptions,\n completion: openai.chat.completions,\n requestBodyParams: prepared.input.requestParams,\n extractContentAndReasoning:\n prepared.input.extractContentAndReasoning,\n useReasoningAsContentFallback:\n prepared.input.useReasoningAsContentFallback,\n },\n isStreaming,\n );\n const { reasoningContent, ...result } = response;\n requestSignal.throwIfAborted();\n const timeCost = Date.now() - startTime;\n debugCall(`response reasoning content: ${reasoningContent}`);\n debugCall(`response content: ${result.content}`);\n recordEvent?.({\n type: 'response',\n http: openAIRequestContext.httpResponse,\n final: {\n content: result.content,\n reasoningContent,\n usage: result.rawUsage,\n requestId: result.requestId,\n timeCost,\n responseModelName: result.responseModelName,\n },\n });\n return {\n ...result,\n reasoning_content: reasoningContent,\n isStreamed: isStreaming,\n timeCost,\n };\n } catch (error: any) {\n // Fetch already records non-2xx responses. Preserve successful HTTP\n // responses too when SDK parsing or completion validation fails.\n if (openAIRequestContext.httpResponse?.ok) {\n recordEvent?.({\n type: 'error',\n ...openAIRequestContext.httpResponse,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n getDebug('ai:call', { console: true })('call AI error', error);\n if (error instanceof AIResponseParseError) {\n throw error;\n }\n throw new Error(\n `failed to call ${isStreaming ? 'streaming ' : ''}AI model service (${modelName}): ${error.message}${formatOpenAIAPIErrorDetails(error, openAIRequestContext)}\\nTrouble shooting: https://midscenejs.com/model-~~provider.html~~`,\n { cause: error },\n );\n }\n}\n"],"names":["callOpenAI","modelRuntime","options","executionId","recordEvent","requestSignal","effectiveTimeoutMs","prepared","modelConfig","modelName","isStreaming","openai","openAIRequestContext","createClient","debugCall","getDebug","requestOptions","undefined","startTime","Date","response","callResponses","callChatCompletion","reasoningContent","result","timeCost","error","Error","String","AIResponseParseError","formatOpenAIAPIErrorDetails"],"mappings":";;;;;;AAqBO,eAAeA,WACpB,EACEC,YAAY,EACZC,OAAO,EACPC,WAAW,EACXC,WAAW,EACXC,aAAa,EACbC,kBAAkB,EACD,EACnBC,QAA6B;IAE7B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGP;IAChC,MAAM,EAAEQ,SAAS,EAAE,GAAGD;IACtB,MAAME,cAAcR,SAAS,UAAUM,AAA2B,aAA3BA,YAAY,UAAU;IAC7D,MAAM,EAAEG,MAAM,EAAEC,oBAAoB,EAAE,GAAG,MAAMC,aAAa;QAC1DL;QACA,YAAYD,SAAS,KAAK,CAAC,UAAU;QACrCD;QACAH;QACAC;IACF;IACA,MAAMU,YAAYC,SAAS;IAC3BD,UACE,CAAC,QAAQ,EAAEJ,cAAc,eAAe,GAAG,WAAW,EAAED,WAAW;IAErE,MAAMO,iBAAiB;QACrBX;QACAO;QACA,SAASV,SAAS,WAAa,KAAKe,MAAQ;QAC5Cb;IACF;IACA,IAAI;QACF,MAAMc,YAAYC,KAAK,GAAG;QAC1B,MAAMC,WACJb,AAAsB,gBAAtBA,SAAS,QAAQ,GACb,MAAMc,cACJ;YACE,GAAGL,cAAc;YACjB,WAAWL,OAAO,SAAS;YAC3B,mBAAmBJ,SAAS,KAAK,CAAC,aAAa;QACjD,GACAG,eAEF,MAAMY,mBACJ;YACE,GAAGN,cAAc;YACjB,YAAYL,OAAO,IAAI,CAAC,WAAW;YACnC,mBAAmBJ,SAAS,KAAK,CAAC,aAAa;YAC/C,4BACEA,SAAS,KAAK,CAAC,0BAA0B;YAC3C,+BACEA,SAAS,KAAK,CAAC,6BAA6B;QAChD,GACAG;QAER,MAAM,EAAEa,gBAAgB,EAAE,GAAGC,QAAQ,GAAGJ;QACxCf,cAAc,cAAc;QAC5B,MAAMoB,WAAWN,KAAK,GAAG,KAAKD;QAC9BJ,UAAU,CAAC,4BAA4B,EAAES,kBAAkB;QAC3DT,UAAU,CAAC,kBAAkB,EAAEU,OAAO,OAAO,EAAE;QAC/CpB,cAAc;YACZ,MAAM;YACN,MAAMQ,qBAAqB,YAAY;YACvC,OAAO;gBACL,SAASY,OAAO,OAAO;gBACvBD;gBACA,OAAOC,OAAO,QAAQ;gBACtB,WAAWA,OAAO,SAAS;gBAC3BC;gBACA,mBAAmBD,OAAO,iBAAiB;YAC7C;QACF;QACA,OAAO;YACL,GAAGA,MAAM;YACT,mBAAmBD;YACnB,YAAYb;YACZe;QACF;IACF,EAAE,OAAOC,OAAY;QAGnB,IAAId,qBAAqB,YAAY,EAAE,IACrCR,cAAc;YACZ,MAAM;YACN,GAAGQ,qBAAqB,YAAY;YACpC,OAAOc,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QACzD;QAEFX,SAAS,WAAW;~~YAAE~~,SAAS;~~QAAK~~,GAAG,~~iBAAiBW~~;~~QACxD~~,IAAIA,iBAAiBG,sBACnB,MAAMH;QAER,MAAM,IAAIC,MACR,CAAC,eAAe,EAAEjB,cAAc,eAAe,GAAG,kBAAkB,EAAED,UAAU,GAAG,EAAEiB,MAAM,OAAO,GAAGI,4BAA4BJ,OAAOd,sBAAsB,~~8DAA8D~~,CAAC,~~EAC7N~~;YAAE,OAAOc;QAAM;IAEnB;AACF"}
1	+ {"version":3,"file":"ai-model/service-caller/openai/call-openai.mjs","sources":["../../../../../src/ai-model/service-caller/openai/call-openai.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport type { ModelCallContext, ModelCallResult } from '../types';\nimport { AIResponseParseError } from '../utils';\nimport {\n callChatCompletion,\n type prepareChatCompletion,\n} from './chat-completion/chat-completion';\nimport { createClient } from './openai-client';\nimport { formatOpenAIAPIErrorDetails } from './openai-request-context';\nimport { callResponses, type prepareResponses } from './responses/responses';\n\nexport type PreparedOpenAIInput =\n \| {\n protocol: 'chat-completion';\n input: Awaited<ReturnType<typeof prepareChatCompletion>>;\n }\n \| {\n protocol: 'responses';\n input: Awaited<ReturnType<typeof prepareResponses>>;\n };\n\nexport async function callOpenAI(\n {\n modelRuntime,\n options,\n executionId,\n recordEvent,\n requestSignal,\n effectiveTimeoutMs,\n }: ModelCallContext,\n prepared: PreparedOpenAIInput,\n): Promise<ModelCallResult> {\n const { config: modelConfig } = modelRuntime;\n const { modelName } = modelConfig;\n const isStreaming = options?.stream ?? modelConfig.streamMode === 'stream';\n const { openai, openAIRequestContext } = await createClient({\n modelConfig,\n proxyAgent: prepared.input.proxyAgent,\n effectiveTimeoutMs,\n executionId,\n recordEvent,\n });\n const debugCall = getDebug('ai:call');\n debugCall(\n `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n );\n const requestOptions = {\n requestSignal,\n openAIRequestContext,\n onChunk: options?.onChunk ?? (() => undefined),\n recordEvent,\n };\n try {\n const startTime = Date.now();\n const response =\n prepared.protocol === 'responses'\n ? await callResponses(\n {\n ...requestOptions,\n responses: openai.responses,\n requestBodyParams: prepared.input.requestParams,\n },\n isStreaming,\n )\n : await callChatCompletion(\n {\n ...requestOptions,\n completion: openai.chat.completions,\n requestBodyParams: prepared.input.requestParams,\n extractContentAndReasoning:\n prepared.input.extractContentAndReasoning,\n useReasoningAsContentFallback:\n prepared.input.useReasoningAsContentFallback,\n },\n isStreaming,\n );\n const { reasoningContent, ...result } = response;\n requestSignal.throwIfAborted();\n const timeCost = Date.now() - startTime;\n debugCall(`response reasoning content: ${reasoningContent}`);\n debugCall(`response content: ${result.content}`);\n recordEvent?.({\n type: 'response',\n http: openAIRequestContext.httpResponse,\n final: {\n content: result.content,\n reasoningContent,\n usage: result.rawUsage,\n requestId: result.requestId,\n timeCost,\n responseModelName: result.responseModelName,\n },\n });\n return {\n ...result,\n reasoning_content: reasoningContent,\n isStreamed: isStreaming,\n timeCost,\n };\n } catch (error: any) {\n // Fetch already records non-2xx responses. Preserve successful HTTP\n // responses too when SDK parsing or completion validation fails.\n if (openAIRequestContext.httpResponse?.ok) {\n recordEvent?.({\n type: 'error',\n ...openAIRequestContext.httpResponse,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n getDebug('ai:call', {\n console: true,\n warning: { kind: 'model-request-detail', modelName, error },\n })('call AI error', error);\n if (error instanceof AIResponseParseError) {\n throw error;\n }\n throw new Error(\n `failed to call ${isStreaming ? 'streaming ' : ''}AI model service (${modelName}): ${error.message}${formatOpenAIAPIErrorDetails(error, openAIRequestContext)}\\nTrouble shooting: https://midscenejs.com/model-config#verify-model-connectivity`,\n { cause: error },\n );\n }\n}\n"],"names":["callOpenAI","modelRuntime","options","executionId","recordEvent","requestSignal","effectiveTimeoutMs","prepared","modelConfig","modelName","isStreaming","openai","openAIRequestContext","createClient","debugCall","getDebug","requestOptions","undefined","startTime","Date","response","callResponses","callChatCompletion","reasoningContent","result","timeCost","error","Error","String","AIResponseParseError","formatOpenAIAPIErrorDetails"],"mappings":";;;;;;AAqBO,eAAeA,WACpB,EACEC,YAAY,EACZC,OAAO,EACPC,WAAW,EACXC,WAAW,EACXC,aAAa,EACbC,kBAAkB,EACD,EACnBC,QAA6B;IAE7B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGP;IAChC,MAAM,EAAEQ,SAAS,EAAE,GAAGD;IACtB,MAAME,cAAcR,SAAS,UAAUM,AAA2B,aAA3BA,YAAY,UAAU;IAC7D,MAAM,EAAEG,MAAM,EAAEC,oBAAoB,EAAE,GAAG,MAAMC,aAAa;QAC1DL;QACA,YAAYD,SAAS,KAAK,CAAC,UAAU;QACrCD;QACAH;QACAC;IACF;IACA,MAAMU,YAAYC,SAAS;IAC3BD,UACE,CAAC,QAAQ,EAAEJ,cAAc,eAAe,GAAG,WAAW,EAAED,WAAW;IAErE,MAAMO,iBAAiB;QACrBX;QACAO;QACA,SAASV,SAAS,WAAa,KAAKe,MAAQ;QAC5Cb;IACF;IACA,IAAI;QACF,MAAMc,YAAYC,KAAK,GAAG;QAC1B,MAAMC,WACJb,AAAsB,gBAAtBA,SAAS,QAAQ,GACb,MAAMc,cACJ;YACE,GAAGL,cAAc;YACjB,WAAWL,OAAO,SAAS;YAC3B,mBAAmBJ,SAAS,KAAK,CAAC,aAAa;QACjD,GACAG,eAEF,MAAMY,mBACJ;YACE,GAAGN,cAAc;YACjB,YAAYL,OAAO,IAAI,CAAC,WAAW;YACnC,mBAAmBJ,SAAS,KAAK,CAAC,aAAa;YAC/C,4BACEA,SAAS,KAAK,CAAC,0BAA0B;YAC3C,+BACEA,SAAS,KAAK,CAAC,6BAA6B;QAChD,GACAG;QAER,MAAM,EAAEa,gBAAgB,EAAE,GAAGC,QAAQ,GAAGJ;QACxCf,cAAc,cAAc;QAC5B,MAAMoB,WAAWN,KAAK,GAAG,KAAKD;QAC9BJ,UAAU,CAAC,4BAA4B,EAAES,kBAAkB;QAC3DT,UAAU,CAAC,kBAAkB,EAAEU,OAAO,OAAO,EAAE;QAC/CpB,cAAc;YACZ,MAAM;YACN,MAAMQ,qBAAqB,YAAY;YACvC,OAAO;gBACL,SAASY,OAAO,OAAO;gBACvBD;gBACA,OAAOC,OAAO,QAAQ;gBACtB,WAAWA,OAAO,SAAS;gBAC3BC;gBACA,mBAAmBD,OAAO,iBAAiB;YAC7C;QACF;QACA,OAAO;YACL,GAAGA,MAAM;YACT,mBAAmBD;YACnB,YAAYb;YACZe;QACF;IACF,EAAE,OAAOC,OAAY;QAGnB,IAAId,qBAAqB,YAAY,EAAE,IACrCR,cAAc;YACZ,MAAM;YACN,GAAGQ,qBAAqB,YAAY;YACpC,OAAOc,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QACzD;QAEFX,SAAS,WAAW;YAClB,SAAS;YACT,SAAS;gBAAE,MAAM;gBAAwBN;gBAAWiB;YAAM;QAC5D,GAAG,iBAAiBA;QACpB,IAAIA,iBAAiBG,sBACnB,MAAMH;QAER,MAAM,IAAIC,MACR,CAAC,eAAe,EAAEjB,cAAc,eAAe,GAAG,kBAAkB,EAAED,UAAU,GAAG,EAAEiB,MAAM,OAAO,GAAGI,4BAA4BJ,OAAOd,sBAAsB,iFAAiF,CAAC,EAChP;YAAE,OAAOc;QAAM;IAEnB;AACF"}

package/dist/es/ai-model/shared/model-locate-result/errors.mjs CHANGED Viewed

@@ -1,4 +1,4 @@
-const modelConfigDocUrl = 'https://midscenejs.com/model-config';
+const modelConfigDocUrl = 'https://midscenejs.com/model-config#common-configuration-errors';
 function modelFamilyRequiredForLocateMessage(slot) {
     const role = 'locate' === slot ? 'Locate' : 'Default';
     const configKey = 'locate' === slot ? 'MIDSCENE_LOCATE_MODEL_FAMILY' : 'MIDSCENE_MODEL_FAMILY';

package/dist/es/ai-model/shared/model-locate-result/errors.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/shared/model-locate-result/errors.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/errors.ts"],"sourcesContent":["const modelConfigDocUrl = 'https://midscenejs.com/model-config';\n\nexport function modelFamilyRequiredForLocateMessage(slot?: string) {\n const role = slot === 'locate' ? 'Locate' : 'Default';\n const configKey =\n slot === 'locate'\n ? 'MIDSCENE_LOCATE_MODEL_FAMILY'\n : 'MIDSCENE_MODEL_FAMILY';\n return `${role} model family is required for locate. Configure ${configKey} so Midscene can parse locate coordinates correctly. ${modelConfigDocUrl}`;\n}\n\nexport function planningModelFamilyRequiredForLocateMessage(slot?: string) {\n if (slot === 'planning') {\n return `Planning model family is required because aiAct is asking the planning model to return locate coordinates. Configure MIDSCENE_PLANNING_MODEL_FAMILY for the planning model, or remove the separate planning model config and configure MIDSCENE_MODEL_FAMILY on the default model. ${modelConfigDocUrl}`;\n }\n\n return `Default model family is required because aiAct is asking the default model to return locate coordinates during planning. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse planning locate coordinates correctly. ${modelConfigDocUrl}`;\n}\n"],"names":["modelConfigDocUrl","modelFamilyRequiredForLocateMessage","slot","role","configKey","planningModelFamilyRequiredForLocateMessage"],"mappings":"AAAA,MAAMA,~~oBAAoB~~;~~AAEnB~~,SAASC,oCAAoCC,IAAa;IAC/D,MAAMC,OAAOD,AAAS,aAATA,OAAoB,WAAW;IAC5C,MAAME,YACJF,AAAS,aAATA,OACI,iCACA;IACN,OAAO,GAAGC,KAAK,gDAAgD,EAAEC,UAAU,qDAAqD,EAAEJ,mBAAmB;AACvJ;AAEO,SAASK,4CAA4CH,IAAa;IACvE,IAAIA,AAAS,eAATA,MACF,OAAO,CAAC,mRAAmR,EAAEF,mBAAmB;IAGlT,OAAO,CAAC,sNAAsN,EAAEA,mBAAmB;AACrP"}
1	+ {"version":3,"file":"ai-model/shared/model-locate-result/errors.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/errors.ts"],"sourcesContent":["const modelConfigDocUrl =\n 'https://midscenejs.com/model-config#common-configuration-errors';\n\nexport function modelFamilyRequiredForLocateMessage(slot?: string) {\n const role = slot === 'locate' ? 'Locate' : 'Default';\n const configKey =\n slot === 'locate'\n ? 'MIDSCENE_LOCATE_MODEL_FAMILY'\n : 'MIDSCENE_MODEL_FAMILY';\n return `${role} model family is required for locate. Configure ${configKey} so Midscene can parse locate coordinates correctly. ${modelConfigDocUrl}`;\n}\n\nexport function planningModelFamilyRequiredForLocateMessage(slot?: string) {\n if (slot === 'planning') {\n return `Planning model family is required because aiAct is asking the planning model to return locate coordinates. Configure MIDSCENE_PLANNING_MODEL_FAMILY for the planning model, or remove the separate planning model config and configure MIDSCENE_MODEL_FAMILY on the default model. ${modelConfigDocUrl}`;\n }\n\n return `Default model family is required because aiAct is asking the default model to return locate coordinates during planning. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse planning locate coordinates correctly. ${modelConfigDocUrl}`;\n}\n"],"names":["modelConfigDocUrl","modelFamilyRequiredForLocateMessage","slot","role","configKey","planningModelFamilyRequiredForLocateMessage"],"mappings":"AAAA,MAAMA,oBACJ;AAEK,SAASC,oCAAoCC,IAAa;IAC/D,MAAMC,OAAOD,AAAS,aAATA,OAAoB,WAAW;IAC5C,MAAME,YACJF,AAAS,aAATA,OACI,iCACA;IACN,OAAO,GAAGC,KAAK,gDAAgD,EAAEC,UAAU,qDAAqD,EAAEJ,mBAAmB;AACvJ;AAEO,SAASK,4CAA4CH,IAAa;IACvE,IAAIA,AAAS,eAATA,MACF,OAAO,CAAC,mRAAmR,EAAEF,mBAAmB;IAGlT,OAAO,CAAC,sNAAsN,EAAEA,mBAAmB;AACrP"}

package/dist/es/ai-model/shared/model-locate-result/parse.mjs CHANGED Viewed

@@ -8,7 +8,11 @@ function parseCoordinateList(input, label) {
     const unwrapped = unwrapCoordinateListLikeInput(input);
     const values = 'string' == typeof unwrapped ? unwrapped.trim().split(/[\s,]+/).filter(Boolean) : unwrapped;
     if (!Array.isArray(values)) throw new Error(`invalid ${label} data: ${JSON.stringify(input)} `);
-    const numericValues = values.map((value)=>'number' == typeof value ? value : Number(value));
+    const numericValues = values.map((value)=>{
+        if ('number' == typeof value) return value;
+        if ('string' != typeof value || !/^[+-]?(?:\d+\.?\d*|\.\d+)$/.test(value.trim())) return NaN;
+        return Number(value);
+    });
     if (!numericValues.every((value)=>Number.isFinite(value))) throw new Error(`invalid ${label} data: ${JSON.stringify(input)} `);
     return numericValues;
 }
@@ -33,7 +37,7 @@ function createLocateResultValue(coordinatesMeta, coordinates) {
 function parseNumericLocateResult(resolvedCoordinates, input) {
     if ('point' === resolvedCoordinates.shape) {
         const point = parseCoordinateList(input, 'point');
-        if (point.length < 2) throw new Error(`invalid point data: ${JSON.stringify(input)} `);
+        if (2 !== point.length) throw new Error(`invalid point data: ${JSON.stringify(input)} `);
         return createLocateResultValue(resolvedCoordinates, point);
     }
     const bbox = parseCoordinateList(input, 'bbox');

package/dist/es/ai-model/shared/model-locate-result/parse.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/shared/model-locate-result/parse.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/parse.ts"],"sourcesContent":["import type {\n BboxLocateResultCoordinates,\n BboxLocateResultValue,\n LocateResultBbox,\n LocateResultPoint,\n LocateResultValue,\n PointLocateResultCoordinates,\n PointLocateResultValue,\n ResolvedLocateResultCoordinates,\n} from './types';\n\ntype CoordinateListLikeInput =\n \| number[]\n \| string[]\n \| string\n \| (number[] \| string[])[];\n\nexport function unwrapCoordinateListLikeInput(\n coordinateList: CoordinateListLikeInput,\n): number[] \| string[] \| string {\n if (Array.isArray(coordinateList)) {\n if (Array.isArray(coordinateList[0])) {\n return coordinateList[0] as number[] \| string[];\n }\n return coordinateList as number[] \| string[];\n }\n return coordinateList as string;\n}\n\nexport function parseCoordinateList(input: unknown, label: string): number[] {\n const unwrapped = unwrapCoordinateListLikeInput(\n input as CoordinateListLikeInput,\n );\n const values =\n typeof unwrapped === 'string'\n ? unwrapped\n .trim()\n .split(/[\\s,]+/)\n .filter(Boolean)\n : unwrapped;\n\n if (!Array.isArray(values)) {\n throw new Error(`invalid ${label} data: ${JSON.stringify(input)} `);\n }\n\n const numericValues = values.map((value) ~~=>\~~n typeof value === 'number' ? value : Number(value),\n );\n\n if (!numericValues.every((value) => Number.isFinite(value))) {\n throw new Error(`invalid ${label} data: ${JSON.stringify(input)} `);\n }\n\n return numericValues;\n}\n\nexport function createLocateResultValue(\n coordinatesMeta: PointLocateResultCoordinates,\n coordinates: number[],\n): PointLocateResultValue;\nexport function createLocateResultValue(\n coordinatesMeta: BboxLocateResultCoordinates,\n coordinates: number[],\n): BboxLocateResultValue;\nexport function createLocateResultValue(\n coordinatesMeta: ResolvedLocateResultCoordinates,\n coordinates: number[],\n): LocateResultValue {\n if (coordinatesMeta.shape === 'point') {\n return {\n coordinates: [coordinates[0], coordinates[1]] as LocateResultPoint,\n coordinatesMeta,\n };\n }\n\n return {\n coordinates: [\n coordinates[0],\n coordinates[1],\n coordinates[2],\n coordinates[3],\n ] as LocateResultBbox,\n coordinatesMeta,\n };\n}\n\nexport function parseNumericLocateResult(\n resolvedCoordinates: ResolvedLocateResultCoordinates,\n input: unknown,\n): LocateResultValue {\n if (resolvedCoordinates.shape === 'point') {\n const point = parseCoordinateList(input, 'point');\n if (point.length < 2) {\n throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n }\n return createLocateResultValue(resolvedCoordinates, point);\n }\n\n const bbox = parseCoordinateList(input, 'bbox');\n if (bbox.length !== 4) {\n throw new Error(`invalid bbox data: ${JSON.stringify(input)} `);\n }\n\n return createLocateResultValue(resolvedCoordinates, bbox);\n}\n"],"names":["unwrapCoordinateListLikeInput","coordinateList","Array","parseCoordinateList","input","label","unwrapped","values","Boolean","Error","JSON","numericValues","value","Number","createLocateResultValue","coordinatesMeta","coordinates","parseNumericLocateResult","resolvedCoordinates","point","bbox"],"mappings":"AAiBO,SAASA,8BACdC,cAAuC;IAEvC,IAAIC,MAAM,OAAO,CAACD,iBAChB;QAAA,IAAIC,MAAM,OAAO,CAACD,cAAc,CAAC,EAAE,GACjC,OAAOA,cAAc,CAAC,EAAE;IAC1B;IAGF,OAAOA;AACT;AAEO,SAASE,oBAAoBC,KAAc,EAAEC,KAAa;IAC/D,MAAMC,YAAYN,8BAChBI;IAEF,MAAMG,SACJ,AAAqB,YAArB,OAAOD,YACHA,UACG,IAAI,GACJ,KAAK,CAAC,UACN,MAAM,CAACE,WACVF;IAEN,IAAI,CAACJ,MAAM,OAAO,CAACK,SACjB,MAAM,IAAIE,MAAM,CAAC,QAAQ,EAAEJ,MAAM,OAAO,EAAEK,KAAK,SAAS,CAACN,OAAO,CAAC,CAAC;IAGpE,MAAMO,gBAAgBJ,OAAO,GAAG,CAAC,CAACK,QAChC,AAAiB,YAAjB,OAAOA,~~QAAqBA~~,~~QAAQC~~,OAAOD;~~IAG7C~~,IAAI,CAACD,cAAc,KAAK,CAAC,CAACC,QAAUC,OAAO,QAAQ,CAACD,SAClD,MAAM,IAAIH,MAAM,CAAC,QAAQ,EAAEJ,MAAM,OAAO,EAAEK,KAAK,SAAS,CAACN,OAAO,CAAC,CAAC;IAGpE,OAAOO;AACT;AAUO,SAASG,wBACdC,eAAgD,EAChDC,WAAqB;IAErB,IAAID,AAA0B,YAA1BA,gBAAgB,KAAK,EACvB,OAAO;QACL,aAAa;YAACC,WAAW,CAAC,EAAE;YAAEA,WAAW,CAAC,EAAE;SAAC;QAC7CD;IACF;IAGF,OAAO;QACL,aAAa;YACXC,WAAW,CAAC,EAAE;YACdA,WAAW,CAAC,EAAE;YACdA,WAAW,CAAC,EAAE;YACdA,WAAW,CAAC,EAAE;SACf;QACDD;IACF;AACF;AAEO,SAASE,yBACdC,mBAAoD,EACpDd,KAAc;IAEd,IAAIc,AAA8B,YAA9BA,oBAAoB,KAAK,EAAc;QACzC,MAAMC,QAAQhB,oBAAoBC,OAAO;QACzC,IAAIe,MAAM,MAAM,~~GAAG~~,~~GACjB,~~MAAM,IAAIV,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACN,OAAO,CAAC,CAAC;QAEjE,OAAOU,wBAAwBI,qBAAqBC;IACtD;IAEA,MAAMC,OAAOjB,oBAAoBC,OAAO;IACxC,IAAIgB,AAAgB,MAAhBA,KAAK,MAAM,EACb,MAAM,IAAIX,MAAM,CAAC,mBAAmB,EAAEC,KAAK,SAAS,CAACN,OAAO,CAAC,CAAC;IAGhE,OAAOU,wBAAwBI,qBAAqBE;AACtD"}
1	+ {"version":3,"file":"ai-model/shared/model-locate-result/parse.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/parse.ts"],"sourcesContent":["import type {\n BboxLocateResultCoordinates,\n BboxLocateResultValue,\n LocateResultBbox,\n LocateResultPoint,\n LocateResultValue,\n PointLocateResultCoordinates,\n PointLocateResultValue,\n ResolvedLocateResultCoordinates,\n} from './types';\n\ntype CoordinateListLikeInput =\n \| number[]\n \| string[]\n \| string\n \| (number[] \| string[])[];\n\nexport function unwrapCoordinateListLikeInput(\n coordinateList: CoordinateListLikeInput,\n): number[] \| string[] \| string {\n if (Array.isArray(coordinateList)) {\n if (Array.isArray(coordinateList[0])) {\n return coordinateList[0] as number[] \| string[];\n }\n return coordinateList as number[] \| string[];\n }\n return coordinateList as string;\n}\n\nexport function parseCoordinateList(input: unknown, label: string): number[] {\n const unwrapped = unwrapCoordinateListLikeInput(\n input as CoordinateListLikeInput,\n );\n const values =\n typeof unwrapped === 'string'\n ? unwrapped\n .trim()\n .split(/[\\s,]+/)\n .filter(Boolean)\n : unwrapped;\n\n if (!Array.isArray(values)) {\n throw new Error(`invalid ${label} data: ${JSON.stringify(input)} `);\n }\n\n const numericValues = values.map((value) => {\n if (typeof value === 'number') {\n return value;\n }\n if (\n typeof value !== 'string' \|\|\n !/^[+-]?(?:\\d+\\.?\\d*\|\\.\\d+)$/.test(value.trim())\n ) {\n return Number.NaN;\n }\n return Number(value);\n });\n\n if (!numericValues.every((value) => Number.isFinite(value))) {\n throw new Error(`invalid ${label} data: ${JSON.stringify(input)} `);\n }\n\n return numericValues;\n}\n\nexport function createLocateResultValue(\n coordinatesMeta: PointLocateResultCoordinates,\n coordinates: number[],\n): PointLocateResultValue;\nexport function createLocateResultValue(\n coordinatesMeta: BboxLocateResultCoordinates,\n coordinates: number[],\n): BboxLocateResultValue;\nexport function createLocateResultValue(\n coordinatesMeta: ResolvedLocateResultCoordinates,\n coordinates: number[],\n): LocateResultValue {\n if (coordinatesMeta.shape === 'point') {\n return {\n coordinates: [coordinates[0], coordinates[1]] as LocateResultPoint,\n coordinatesMeta,\n };\n }\n\n return {\n coordinates: [\n coordinates[0],\n coordinates[1],\n coordinates[2],\n coordinates[3],\n ] as LocateResultBbox,\n coordinatesMeta,\n };\n}\n\nexport function parseNumericLocateResult(\n resolvedCoordinates: ResolvedLocateResultCoordinates,\n input: unknown,\n): LocateResultValue {\n if (resolvedCoordinates.shape === 'point') {\n const point = parseCoordinateList(input, 'point');\n if (point.length !== 2) {\n throw new Error(`invalid point data: ${JSON.stringify(input)} `);\n }\n return createLocateResultValue(resolvedCoordinates, point);\n }\n\n const bbox = parseCoordinateList(input, 'bbox');\n if (bbox.length !== 4) {\n throw new Error(`invalid bbox data: ${JSON.stringify(input)} `);\n }\n\n return createLocateResultValue(resolvedCoordinates, bbox);\n}\n"],"names":["unwrapCoordinateListLikeInput","coordinateList","Array","parseCoordinateList","input","label","unwrapped","values","Boolean","Error","JSON","numericValues","value","Number","createLocateResultValue","coordinatesMeta","coordinates","parseNumericLocateResult","resolvedCoordinates","point","bbox"],"mappings":"AAiBO,SAASA,8BACdC,cAAuC;IAEvC,IAAIC,MAAM,OAAO,CAACD,iBAChB;QAAA,IAAIC,MAAM,OAAO,CAACD,cAAc,CAAC,EAAE,GACjC,OAAOA,cAAc,CAAC,EAAE;IAC1B;IAGF,OAAOA;AACT;AAEO,SAASE,oBAAoBC,KAAc,EAAEC,KAAa;IAC/D,MAAMC,YAAYN,8BAChBI;IAEF,MAAMG,SACJ,AAAqB,YAArB,OAAOD,YACHA,UACG,IAAI,GACJ,KAAK,CAAC,UACN,MAAM,CAACE,WACVF;IAEN,IAAI,CAACJ,MAAM,OAAO,CAACK,SACjB,MAAM,IAAIE,MAAM,CAAC,QAAQ,EAAEJ,MAAM,OAAO,EAAEK,KAAK,SAAS,CAACN,OAAO,CAAC,CAAC;IAGpE,MAAMO,gBAAgBJ,OAAO,GAAG,CAAC,CAACK;QAChC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;QAET,IACE,AAAiB,YAAjB,OAAOA,SACP,CAAC,6BAA6B,IAAI,CAACA,MAAM,IAAI,KAE7C,OAAOC;QAET,OAAOA,OAAOD;IAChB;IAEA,IAAI,CAACD,cAAc,KAAK,CAAC,CAACC,QAAUC,OAAO,QAAQ,CAACD,SAClD,MAAM,IAAIH,MAAM,CAAC,QAAQ,EAAEJ,MAAM,OAAO,EAAEK,KAAK,SAAS,CAACN,OAAO,CAAC,CAAC;IAGpE,OAAOO;AACT;AAUO,SAASG,wBACdC,eAAgD,EAChDC,WAAqB;IAErB,IAAID,AAA0B,YAA1BA,gBAAgB,KAAK,EACvB,OAAO;QACL,aAAa;YAACC,WAAW,CAAC,EAAE;YAAEA,WAAW,CAAC,EAAE;SAAC;QAC7CD;IACF;IAGF,OAAO;QACL,aAAa;YACXC,WAAW,CAAC,EAAE;YACdA,WAAW,CAAC,EAAE;YACdA,WAAW,CAAC,EAAE;YACdA,WAAW,CAAC,EAAE;SACf;QACDD;IACF;AACF;AAEO,SAASE,yBACdC,mBAAoD,EACpDd,KAAc;IAEd,IAAIc,AAA8B,YAA9BA,oBAAoB,KAAK,EAAc;QACzC,MAAMC,QAAQhB,oBAAoBC,OAAO;QACzC,IAAIe,AAAiB,MAAjBA,MAAM,MAAM,EACd,MAAM,IAAIV,MAAM,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACN,OAAO,CAAC,CAAC;QAEjE,OAAOU,wBAAwBI,qBAAqBC;IACtD;IAEA,MAAMC,OAAOjB,oBAAoBC,OAAO;IACxC,IAAIgB,AAAgB,MAAhBA,KAAK,MAAM,EACb,MAAM,IAAIX,MAAM,CAAC,mBAAmB,EAAEC,KAAK,SAAS,CAACN,OAAO,CAAC,CAAC;IAGhE,OAAOU,wBAAwBI,qBAAqBE;AACtD"}

package/dist/es/device/index.mjs CHANGED Viewed

@@ -301,16 +301,20 @@ function createActionSwipeParamSchema(inputMode) {
             'left',
             'right'
         ]).optional().describe(`${'finger' === movementSource ? 'Finger' : 'Pointer'} movement direction. Required together with a positive distance for a relative swipe. Omit when using end.`),
-        distance: z.number().positive().optional().describe(`Positive length of the ${movementSource} movement in pixels. Required together with direction for a relative swipe. Omit when using end.`),
+        distance: z.number().finite().positive().optional().describe(`Positive length of the ${movementSource} movement in pixels. Required together with direction for a relative swipe. Omit when using end.`),
         end: getMidsceneLocationSchema().optional().describe(`Endpoint of the ${movementSource} movement. Use for an endpoint swipe, optionally with start. Do not provide direction or distance when using end.`),
-        duration: z.number().default(300).describe('Duration of the swipe gesture in milliseconds'),
-        repeat: z.number().optional().describe('The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)')
+        duration: z.number().finite().positive().default(300).describe('Duration of the swipe gesture in milliseconds'),
+        repeat: z.number().finite().int().nonnegative().optional().describe('The number of times to repeat the swipe gesture. 1 for default, 0 for continuous mode capped at 10 repeats')
     });
 }
 const ActionSwipeParamSchema = createActionSwipeParamSchema('touch');
 function normalizeSwipeParam(param, screenSize) {
     const { width, height } = screenSize;
     const { start, end } = param;
+    const duration = param.duration ?? 300;
+    if (!Number.isFinite(duration) || duration <= 0) throw new Error('duration must be a positive finite number');
+    const repeatCount = param.repeat ?? 1;
+    if (!Number.isFinite(repeatCount) || !Number.isInteger(repeatCount) || repeatCount < 0) throw new Error('repeat must be a non-negative finite integer');
     const startPoint = start ? {
         x: start.center[0],
         y: start.center[1]
@@ -327,7 +331,7 @@ function normalizeSwipeParam(param, screenSize) {
             };
         }
         if (void 0 === param.direction || void 0 === param.distance) throw new Error('Invalid Swipe parameters: a relative swipe requires both "direction" and a positive "distance".');
-        if (param.distance <= 0) throw new Error('Invalid Swipe parameters: "distance" must be a positive number.');
+        if (!Number.isFinite(param.distance) || param.distance <= 0) throw new Error('Invalid Swipe parameters: "distance" must be a positive number.');
         return {
             x: startPoint.x + ('right' === param.direction ? param.distance : 'left' === param.direction ? -param.distance : 0),
             y: startPoint.y + ('down' === param.direction ? param.distance : 'up' === param.direction ? -param.distance : 0)
@@ -335,14 +339,11 @@ function normalizeSwipeParam(param, screenSize) {
     })();
     endPoint.x = Math.max(0, Math.min(endPoint.x, width));
     endPoint.y = Math.max(0, Math.min(endPoint.y, height));
-    const duration = param.duration ?? 300;
-    let repeatCount = 'number' == typeof param.repeat ? param.repeat : 1;
-    if (0 === repeatCount) repeatCount = 10;
     return {
         startPoint,
         endPoint,
         duration,
-        repeatCount
+        repeatCount: 0 === repeatCount ? 10 : repeatCount
     };
 }
 const normalizeMobileSwipeParam = normalizeSwipeParam;