@midscene/core 1.16.2 → 1.16.6-beta-20261023073257.0

package/dist/lib/ai-model/connectivity/run-connectivity-test.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/connectivity/run-connectivity-test.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/connectivity/run-connectivity-test.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { ScreenshotItem } from '@/screenshot-item';\nimport Service from '@/service';\nimport type { UIContext } from '@/types';\nimport type { IModelConfig, TIntent } from '@midscene/shared/env';\nimport { getModelRuntime } from '../models';\nimport type { ModelRuntime } from '../models';\nimport { callAI } from '../service-caller';\nimport { systemOneResponseSchema } from '../service-caller/system-one';\nimport {\n  CONNECTIVITY_FIXTURE_IMAGE,\n  CONNECTIVITY_FIXTURE_SHOT_SIZE,\n} from './fixture';\n\nconst TEXT_EXPECTED_TOKEN = 'CONNECTIVITY_OK';\n\nexport interface ConnectivityCheckResultItem {\n  name: 'text' | 'vision' | 'aiLocate';\n  intent: TIntent;\n  modelName: string;\n  modelFamily?: string;\n  passed: boolean;\n  durationMs: number;\n  requestCount: number;\n  endpoints: string[];\n  message: string;\n}\n\nexport interface ConnectivityTestResult {\n  checks: ConnectivityCheckResultItem[];\n  requestCount: number;\n  passed: boolean;\n  message?: string;\n}\n\nexport interface ConnectivityTestConfig {\n  defaultModelConfig: IModelConfig;\n  locateModelConfig?: IModelConfig;\n  planningModelConfig: IModelConfig;\n  insightModelConfig: IModelConfig;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction hasValidCenter(value: unknown): boolean {\n  return (\n    Array.isArray(value) &&\n    value.length === 2 &&\n    isFiniteNumber(value[0]) &&\n    isFiniteNumber(value[1])\n  );\n}\n\nfunction buildFixtureContext(): UIContext {\n  return {\n    screenshot: ScreenshotItem.create(CONNECTIVITY_FIXTURE_IMAGE, Date.now()),\n    shotSize: CONNECTIVITY_FIXTURE_SHOT_SIZE,\n    shrunkShotToLogicalRatio: 1,\n  };\n}\n\nfunction buildCheckResult(\n  name: ConnectivityCheckResultItem['name'],\n  modelRuntime: ModelRuntime,\n  result: Omit<\n    ConnectivityCheckResultItem,\n    | 'name'\n    | 'intent'\n    | 'modelName'\n    | 'modelFamily'\n    | 'requestCount'\n    | 'endpoints'\n  >,\n): ConnectivityCheckResultItem {\n  const { config } = modelRuntime;\n  return {\n    name,\n    intent: config.intent,\n    modelName: config.modelName,\n    modelFamily: config.modelFamily,\n    requestCount: 0,\n    endpoints: [],\n    ...result,\n  };\n}\n\nfunction buildConnectivityModelRuntime(\n  modelConfig: IModelConfig,\n): ModelRuntime {\n  return getModelRuntime({ ...modelConfig, retryCount: 0 });\n}\n\nfunction formatConnectivityCheckName(\n  check: ConnectivityCheckResultItem,\n): string {\n  const checkName =\n    check.name === 'aiLocate'\n      ? 'AI locate check'\n      : `${check.name[0]?.toUpperCase()}${check.name.slice(1)} check`;\n  const modelLabel = check.modelName || check.intent;\n  return `${checkName} - ${modelLabel} (${check.intent})`;\n}\n\nfunction buildConnectivityFailureMessage(\n  checks: ConnectivityCheckResultItem[],\n): string {\n  const failedChecks = checks.filter((item) => !item.passed);\n  if (failedChecks.length === 0) {\n    return 'Connectivity test failed, but no failed check details were generated.';\n  }\n\n  return failedChecks\n    .map((item) => {\n      const detail = item.message || 'Failed without details.';\n      return `[${formatConnectivityCheckName(item)}]: ${detail}`;\n    })\n    .join('\\n');\n}\n\nasync function runTextConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const isJev = modelRuntime.config.modelFamily === 'jev';\n    const result = await callAI(\n      [\n        {\n          role: 'system',\n          content: 'Reply with the exact token the user asks for.',\n        },\n        {\n          role: 'user',\n          content: `Return exactly ${TEXT_EXPECTED_TOKEN}`,\n        },\n      ],\n      modelRuntime,\n      isJev\n        ? {\n            systemOneRequest: {\n              state: { token: TEXT_EXPECTED_TOKEN },\n              questions: {\n                result: {\n                  type: 'noul',\n                  instructions: `Does token equal ${TEXT_EXPECTED_TOKEN}?`,\n                },\n              },\n            },\n          }\n        : undefined,\n    );\n    const content = result.content.trim();\n    const answer = isJev\n      ? systemOneResponseSchema.parse(JSON.parse(content)).answers.result\n      : undefined;\n    const passed = isJev\n      ? answer?.type === 'noul' && answer.noul >= 0.5\n      : content.includes(TEXT_EXPECTED_TOKEN);\n    return buildCheckResult('text', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed ? '' : `Unexpected response: ${content}`,\n    });\n  } catch (error) {\n    return buildCheckResult('text', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runVisionConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    if (modelRuntime.adapter.insight.booleanFn) {\n      await new Service(buildFixtureContext()).extract(\n        { Boolean: 'Does this screenshot show a form?' },\n        modelRuntime,\n        { screenshotIncluded: true },\n        undefined,\n        undefined,\n        buildFixtureContext(),\n      );\n    } else {\n      await callAI(\n        [\n          {\n            role: 'user',\n            content: [\n              {\n                type: 'text',\n                text: 'What is the main content of this image ? It is a photo or a form ?',\n              },\n              {\n                type: 'image',\n                url: CONNECTIVITY_FIXTURE_IMAGE,\n              },\n            ],\n          },\n        ],\n        modelRuntime,\n      );\n    }\n    return buildCheckResult('vision', modelRuntime, {\n      passed: true,\n      durationMs: Date.now() - startTime,\n      message: '',\n    });\n  } catch (error) {\n    return buildCheckResult('vision', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runAiLocateConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const context = buildFixtureContext();\n    const service = new Service(context);\n    const locateResult = await service.locate(\n      { prompt: 'the main todo input box' },\n      {},\n      modelRuntime,\n    );\n    const center = locateResult.element?.center;\n    const passed = hasValidCenter(center);\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed\n        ? ''\n        : `Invalid locate result: ${JSON.stringify({\n            center,\n          })}`,\n    });\n  } catch (error) {\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nexport async function runConnectivityTest(\n  config: ConnectivityTestConfig,\n): Promise<ConnectivityTestResult> {\n  const runCheck = async (\n    modelConfig: IModelConfig,\n    check: (runtime: ModelRuntime) => Promise<ConnectivityCheckResultItem>,\n  ) => {\n    const runtime = buildConnectivityModelRuntime(modelConfig);\n    let requestCount = 0;\n    const endpoints = new Set<string>();\n    runtime.onRequest = (endpoint) => {\n      requestCount += 1;\n      if (endpoint) endpoints.add(endpoint);\n    };\n    const result = await check(runtime);\n    return { ...result, requestCount, endpoints: [...endpoints] };\n  };\n  const checks = await Promise.all([\n    runCheck(config.planningModelConfig, runTextConnectivityCheck),\n    runCheck(config.insightModelConfig, runVisionConnectivityCheck),\n    runCheck(\n      config.locateModelConfig ?? config.defaultModelConfig,\n      runAiLocateConnectivityCheck,\n    ),\n    runCheck(config.defaultModelConfig, runTextConnectivityCheck),\n  ]);\n\n  const passed = checks.every((item) => item.passed);\n  return {\n    checks,\n    requestCount: checks.reduce(\n      (total, check) => total + check.requestCount,\n      0,\n    ),\n    passed,\n    message: passed ? undefined : buildConnectivityFailureMessage(checks),\n  };\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","TEXT_EXPECTED_TOKEN","isFiniteNumber","value","Number","hasValidCenter","Array","buildFixtureContext","ScreenshotItem","CONNECTIVITY_FIXTURE_IMAGE","Date","CONNECTIVITY_FIXTURE_SHOT_SIZE","buildCheckResult","name","modelRuntime","result","config","buildConnectivityModelRuntime","modelConfig","getModelRuntime","formatConnectivityCheckName","check","checkName","modelLabel","buildConnectivityFailureMessage","checks","failedChecks","item","detail","runTextConnectivityCheck","startTime","isJev","callAI","undefined","content","answer","systemOneResponseSchema","JSON","passed","error","Error","String","runVisionConnectivityCheck","Service","runAiLocateConnectivityCheck","context","service","locateResult","center","runConnectivityTest","runCheck","runtime","requestCount","endpoints","Set","endpoint","Promise","total"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACOA,MAAMI,sBAAsB;AA4B5B,SAASC,eAAeC,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD;AACtD;AAEA,SAASE,eAAeF,KAAc;IACpC,OACEG,MAAM,OAAO,CAACH,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZD,eAAeC,KAAK,CAAC,EAAE,KACvBD,eAAeC,KAAK,CAAC,EAAE;AAE3B;AAEA,SAASI;IACP,OAAO;QACL,YAAYC,4CAAAA,cAAAA,CAAAA,MAAqB,CAACC,oCAAAA,0BAA0BA,EAAEC,KAAK,GAAG;QACtE,UAAUC,oCAAAA,8BAA8BA;QACxC,0BAA0B;IAC5B;AACF;AAEA,SAASC,iBACPC,IAAyC,EACzCC,YAA0B,EAC1BC,MAQC;IAED,MAAM,EAAEC,MAAM,EAAE,GAAGF;IACnB,OAAO;QACLD;QACA,QAAQG,OAAO,MAAM;QACrB,WAAWA,OAAO,SAAS;QAC3B,aAAaA,OAAO,WAAW;QAC/B,cAAc;QACd,WAAW,EAAE;QACb,GAAGD,MAAM;IACX;AACF;AAEA,SAASE,8BACPC,WAAyB;IAEzB,OAAOC,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgB;QAAE,GAAGD,WAAW;QAAE,YAAY;IAAE;AACzD;AAEA,SAASE,4BACPC,KAAkC;IAElC,MAAMC,YACJD,AAAe,eAAfA,MAAM,IAAI,GACN,oBACA,GAAGA,MAAM,IAAI,CAAC,EAAE,EAAE,gBAAgBA,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IACnE,MAAME,aAAaF,MAAM,SAAS,IAAIA,MAAM,MAAM;IAClD,OAAO,GAAGC,UAAU,GAAG,EAAEC,WAAW,EAAE,EAAEF,MAAM,MAAM,CAAC,CAAC,CAAC;AACzD;AAEA,SAASG,gCACPC,MAAqC;IAErC,MAAMC,eAAeD,OAAO,MAAM,CAAC,CAACE,OAAS,CAACA,KAAK,MAAM;IACzD,IAAID,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAO;IAGT,OAAOA,aACJ,GAAG,CAAC,CAACC;QACJ,MAAMC,SAASD,KAAK,OAAO,IAAI;QAC/B,OAAO,CAAC,CAAC,EAAEP,4BAA4BO,MAAM,GAAG,EAAEC,QAAQ;IAC5D,GACC,IAAI,CAAC;AACV;AAEA,eAAeC,yBACbf,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMqB,QAAQjB,AAAoC,UAApCA,aAAa,MAAM,CAAC,WAAW;QAC7C,MAAMC,SAAS,MAAMiB,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACnB;YACE;gBACE,MAAM;gBACN,SAAS;YACX;YACA;gBACE,MAAM;gBACN,SAAS,CAAC,eAAe,EAAE/B,qBAAqB;YAClD;SACD,EACDa,cACAiB,QACI;YACE,kBAAkB;gBAChB,OAAO;oBAAE,OAAO9B;gBAAoB;gBACpC,WAAW;oBACT,QAAQ;wBACN,MAAM;wBACN,cAAc,CAAC,iBAAiB,EAAEA,oBAAoB,CAAC,CAAC;oBAC1D;gBACF;YACF;QACF,IACAgC;QAEN,MAAMC,UAAUnB,OAAO,OAAO,CAAC,IAAI;QACnC,MAAMoB,SAASJ,QACXK,8BAAAA,uBAAAA,CAAAA,KAA6B,CAACC,KAAK,KAAK,CAACH,UAAU,OAAO,CAAC,MAAM,GACjED;QACJ,MAAMK,SAASP,QACXI,QAAQ,SAAS,UAAUA,OAAO,IAAI,IAAI,MAC1CD,QAAQ,QAAQ,CAACjC;QACrB,OAAOW,iBAAiB,QAAQE,cAAc;YAC5CwB;YACA,YAAY5B,KAAK,GAAG,KAAKoB;YACzB,SAASQ,SAAS,KAAK,CAAC,qBAAqB,EAAEJ,SAAS;QAC1D;IACF,EAAE,OAAOK,OAAO;QACd,OAAO3B,iBAAiB,QAAQE,cAAc;YAC5C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASS,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeG,2BACb5B,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,IAAII,aAAa,OAAO,CAAC,OAAO,CAAC,SAAS,EACxC,MAAM,IAAI6B,CAAAA,kBAAAA,EAAQpC,uBAAuB,OAAO,CAC9C;YAAE,SAAS;QAAoC,GAC/CO,cACA;YAAE,oBAAoB;QAAK,GAC3BmB,QACAA,QACA1B;aAGF,MAAMyB,AAAAA,IAAAA,iDAAAA,MAAAA,AAAAA,EACJ;YACE;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM;oBACR;oBACA;wBACE,MAAM;wBACN,KAAKvB,oCAAAA,0BAA0BA;oBACjC;iBACD;YACH;SACD,EACDK;QAGJ,OAAOF,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAAS;QACX;IACF,EAAE,OAAOS,OAAO;QACd,OAAO3B,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASS,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeK,6BACb9B,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMmC,UAAUtC;QAChB,MAAMuC,UAAU,IAAIH,CAAAA,kBAAAA,EAAQE;QAC5B,MAAME,eAAe,MAAMD,QAAQ,MAAM,CACvC;YAAE,QAAQ;QAA0B,GACpC,CAAC,GACDhC;QAEF,MAAMkC,SAASD,aAAa,OAAO,EAAE;QACrC,MAAMT,SAASjC,eAAe2C;QAC9B,OAAOpC,iBAAiB,YAAYE,cAAc;YAChDwB;YACA,YAAY5B,KAAK,GAAG,KAAKoB;YACzB,SAASQ,SACL,KACA,CAAC,uBAAuB,EAAED,KAAK,SAAS,CAAC;gBACvCW;YACF,IAAI;QACV;IACF,EAAE,OAAOT,OAAO;QACd,OAAO3B,iBAAiB,YAAYE,cAAc;YAChD,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASS,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEO,eAAeU,oBACpBjC,MAA8B;IAE9B,MAAMkC,WAAW,OACfhC,aACAG;QAEA,MAAM8B,UAAUlC,8BAA8BC;QAC9C,IAAIkC,eAAe;QACnB,MAAMC,YAAY,IAAIC;QACtBH,QAAQ,SAAS,GAAG,CAACI;YACnBH,gBAAgB;YAChB,IAAIG,UAAUF,UAAU,GAAG,CAACE;QAC9B;QACA,MAAMxC,SAAS,MAAMM,MAAM8B;QAC3B,OAAO;YAAE,GAAGpC,MAAM;YAAEqC;YAAc,WAAW;mBAAIC;aAAU;QAAC;IAC9D;IACA,MAAM5B,SAAS,MAAM+B,QAAQ,GAAG,CAAC;QAC/BN,SAASlC,OAAO,mBAAmB,EAAEa;QACrCqB,SAASlC,OAAO,kBAAkB,EAAE0B;QACpCQ,SACElC,OAAO,iBAAiB,IAAIA,OAAO,kBAAkB,EACrD4B;QAEFM,SAASlC,OAAO,kBAAkB,EAAEa;KACrC;IAED,MAAMS,SAASb,OAAO,KAAK,CAAC,CAACE,OAASA,KAAK,MAAM;IACjD,OAAO;QACLF;QACA,cAAcA,OAAO,MAAM,CACzB,CAACgC,OAAOpC,QAAUoC,QAAQpC,MAAM,YAAY,EAC5C;QAEFiB;QACA,SAASA,SAASL,SAAYT,gCAAgCC;IAChE;AACF"}
+{"version":3,"file":"ai-model/connectivity/run-connectivity-test.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/connectivity/run-connectivity-test.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { ScreenshotItem } from '@/screenshot-item';\nimport Service from '@/service';\nimport type { UIContext } from '@/types';\nimport type { IModelConfig, TIntent } from '@midscene/shared/env';\nimport { getModelRuntime } from '../models';\nimport type { ModelRuntime } from '../models';\nimport { callGenerativeAI, callSystemOneAI } from '../service-caller';\nimport { systemOneResponseSchema } from '../service-caller/system-one/schema';\nimport {\n  CONNECTIVITY_FIXTURE_IMAGE,\n  CONNECTIVITY_FIXTURE_SHOT_SIZE,\n} from './fixture';\n\nconst TEXT_EXPECTED_TOKEN = 'CONNECTIVITY_OK';\n\nexport interface ConnectivityCheckResultItem {\n  name: 'text' | 'vision' | 'aiLocate';\n  intent: TIntent;\n  modelName: string;\n  modelFamily?: string;\n  passed: boolean;\n  durationMs: number;\n  requestCount: number;\n  endpoints: string[];\n  message: string;\n}\n\nexport interface ConnectivityTestResult {\n  checks: ConnectivityCheckResultItem[];\n  requestCount: number;\n  passed: boolean;\n  message?: string;\n}\n\nexport interface ConnectivityTestConfig {\n  defaultModelConfig: IModelConfig;\n  locateModelConfig?: IModelConfig;\n  planningModelConfig: IModelConfig;\n  insightModelConfig: IModelConfig;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction hasValidCenter(value: unknown): boolean {\n  return (\n    Array.isArray(value) &&\n    value.length === 2 &&\n    isFiniteNumber(value[0]) &&\n    isFiniteNumber(value[1])\n  );\n}\n\nfunction buildFixtureContext(): UIContext {\n  return {\n    screenshot: ScreenshotItem.create(CONNECTIVITY_FIXTURE_IMAGE, Date.now()),\n    shotSize: CONNECTIVITY_FIXTURE_SHOT_SIZE,\n    shrunkShotToLogicalRatio: 1,\n  };\n}\n\nfunction buildCheckResult(\n  name: ConnectivityCheckResultItem['name'],\n  modelRuntime: ModelRuntime,\n  result: Omit<\n    ConnectivityCheckResultItem,\n    | 'name'\n    | 'intent'\n    | 'modelName'\n    | 'modelFamily'\n    | 'requestCount'\n    | 'endpoints'\n  >,\n): ConnectivityCheckResultItem {\n  const { config } = modelRuntime;\n  return {\n    name,\n    intent: config.intent,\n    modelName: config.modelName,\n    modelFamily: config.modelFamily,\n    requestCount: 0,\n    endpoints: [],\n    ...result,\n  };\n}\n\nfunction buildConnectivityModelRuntime(\n  modelConfig: IModelConfig,\n): ModelRuntime {\n  return getModelRuntime({ ...modelConfig, retryCount: 0 });\n}\n\nfunction formatConnectivityCheckName(\n  check: ConnectivityCheckResultItem,\n): string {\n  const checkName =\n    check.name === 'aiLocate'\n      ? 'AI locate check'\n      : `${check.name[0]?.toUpperCase()}${check.name.slice(1)} check`;\n  const modelLabel = check.modelName || check.intent;\n  return `${checkName} - ${modelLabel} (${check.intent})`;\n}\n\nfunction buildConnectivityFailureMessage(\n  checks: ConnectivityCheckResultItem[],\n): string {\n  const failedChecks = checks.filter((item) => !item.passed);\n  if (failedChecks.length === 0) {\n    return 'Connectivity test failed, but no failed check details were generated.';\n  }\n\n  return failedChecks\n    .map((item) => {\n      const detail = item.message || 'Failed without details.';\n      return `[${formatConnectivityCheckName(item)}]: ${detail}`;\n    })\n    .join('\\n');\n}\n\nasync function runTextConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const isJev = modelRuntime.config.modelFamily === 'jev';\n    const result = await (isJev\n      ? callSystemOneAI(\n          {\n            state: { token: TEXT_EXPECTED_TOKEN },\n            questions: {\n              result: {\n                type: 'noul',\n                instructions: `Does token equal ${TEXT_EXPECTED_TOKEN}?`,\n              },\n            },\n          },\n          modelRuntime,\n        )\n      : callGenerativeAI(\n          [\n            {\n              role: 'system',\n              content: 'Reply with the exact token the user asks for.',\n            },\n            {\n              role: 'user',\n              content: `Return exactly ${TEXT_EXPECTED_TOKEN}`,\n            },\n          ],\n          modelRuntime,\n        ));\n    const content = result.content.trim();\n    const answer = isJev\n      ? systemOneResponseSchema.parse(JSON.parse(content)).answers.result\n      : undefined;\n    const passed = isJev\n      ? answer?.type === 'noul' && answer.noul >= 0.5\n      : content.includes(TEXT_EXPECTED_TOKEN);\n    return buildCheckResult('text', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed ? '' : `Unexpected response: ${content}`,\n    });\n  } catch (error) {\n    return buildCheckResult('text', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runVisionConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    if (modelRuntime.config.modelFamily === 'jev') {\n      await callSystemOneAI(\n        {\n          state: {},\n          images: { screenshot: CONNECTIVITY_FIXTURE_IMAGE },\n          questions: {\n            result: {\n              type: 'noul',\n              instructions: 'Does `screenshot` show a form?',\n            },\n          },\n        },\n        modelRuntime,\n      );\n    } else {\n      await callGenerativeAI(\n        [\n          {\n            role: 'user',\n            content: [\n              {\n                type: 'text',\n                text: 'What is the main content of this image ? It is a photo or a form ?',\n              },\n              {\n                type: 'image',\n                url: CONNECTIVITY_FIXTURE_IMAGE,\n              },\n            ],\n          },\n        ],\n        modelRuntime,\n      );\n    }\n    return buildCheckResult('vision', modelRuntime, {\n      passed: true,\n      durationMs: Date.now() - startTime,\n      message: '',\n    });\n  } catch (error) {\n    return buildCheckResult('vision', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runAiLocateConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const context = buildFixtureContext();\n    const service = new Service(context);\n    const locateResult = await service.locate(\n      { prompt: 'the main todo input box' },\n      {},\n      modelRuntime,\n    );\n    const center = locateResult.element?.center;\n    const passed = hasValidCenter(center);\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed\n        ? ''\n        : `Invalid locate result: ${JSON.stringify({\n            center,\n          })}`,\n    });\n  } catch (error) {\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nexport async function runConnectivityTest(\n  config: ConnectivityTestConfig,\n): Promise<ConnectivityTestResult> {\n  const runCheck = async (\n    modelConfig: IModelConfig,\n    check: (runtime: ModelRuntime) => Promise<ConnectivityCheckResultItem>,\n  ) => {\n    const runtime = buildConnectivityModelRuntime(modelConfig);\n    let requestCount = 0;\n    const endpoints = new Set<string>();\n    runtime.onRequest = (endpoint) => {\n      requestCount += 1;\n      if (endpoint) endpoints.add(endpoint);\n    };\n    const result = await check(runtime);\n    return { ...result, requestCount, endpoints: [...endpoints] };\n  };\n  const checks = await Promise.all([\n    runCheck(config.planningModelConfig, runTextConnectivityCheck),\n    runCheck(config.insightModelConfig, runVisionConnectivityCheck),\n    runCheck(\n      config.locateModelConfig ?? config.defaultModelConfig,\n      runAiLocateConnectivityCheck,\n    ),\n    runCheck(config.defaultModelConfig, runTextConnectivityCheck),\n  ]);\n\n  const passed = checks.every((item) => item.passed);\n  return {\n    checks,\n    requestCount: checks.reduce(\n      (total, check) => total + check.requestCount,\n      0,\n    ),\n    passed,\n    message: passed ? undefined : buildConnectivityFailureMessage(checks),\n  };\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","TEXT_EXPECTED_TOKEN","isFiniteNumber","value","Number","hasValidCenter","Array","buildFixtureContext","ScreenshotItem","CONNECTIVITY_FIXTURE_IMAGE","Date","CONNECTIVITY_FIXTURE_SHOT_SIZE","buildCheckResult","name","modelRuntime","result","config","buildConnectivityModelRuntime","modelConfig","getModelRuntime","formatConnectivityCheckName","check","checkName","modelLabel","buildConnectivityFailureMessage","checks","failedChecks","item","detail","runTextConnectivityCheck","startTime","isJev","callSystemOneAI","callGenerativeAI","content","answer","systemOneResponseSchema","JSON","undefined","passed","error","Error","String","runVisionConnectivityCheck","runAiLocateConnectivityCheck","context","service","Service","locateResult","center","runConnectivityTest","runCheck","runtime","requestCount","endpoints","Set","endpoint","Promise","total"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACOA,MAAMI,sBAAsB;AA4B5B,SAASC,eAAeC,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD;AACtD;AAEA,SAASE,eAAeF,KAAc;IACpC,OACEG,MAAM,OAAO,CAACH,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZD,eAAeC,KAAK,CAAC,EAAE,KACvBD,eAAeC,KAAK,CAAC,EAAE;AAE3B;AAEA,SAASI;IACP,OAAO;QACL,YAAYC,4CAAAA,cAAAA,CAAAA,MAAqB,CAACC,oCAAAA,0BAA0BA,EAAEC,KAAK,GAAG;QACtE,UAAUC,oCAAAA,8BAA8BA;QACxC,0BAA0B;IAC5B;AACF;AAEA,SAASC,iBACPC,IAAyC,EACzCC,YAA0B,EAC1BC,MAQC;IAED,MAAM,EAAEC,MAAM,EAAE,GAAGF;IACnB,OAAO;QACLD;QACA,QAAQG,OAAO,MAAM;QACrB,WAAWA,OAAO,SAAS;QAC3B,aAAaA,OAAO,WAAW;QAC/B,cAAc;QACd,WAAW,EAAE;QACb,GAAGD,MAAM;IACX;AACF;AAEA,SAASE,8BACPC,WAAyB;IAEzB,OAAOC,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgB;QAAE,GAAGD,WAAW;QAAE,YAAY;IAAE;AACzD;AAEA,SAASE,4BACPC,KAAkC;IAElC,MAAMC,YACJD,AAAe,eAAfA,MAAM,IAAI,GACN,oBACA,GAAGA,MAAM,IAAI,CAAC,EAAE,EAAE,gBAAgBA,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IACnE,MAAME,aAAaF,MAAM,SAAS,IAAIA,MAAM,MAAM;IAClD,OAAO,GAAGC,UAAU,GAAG,EAAEC,WAAW,EAAE,EAAEF,MAAM,MAAM,CAAC,CAAC,CAAC;AACzD;AAEA,SAASG,gCACPC,MAAqC;IAErC,MAAMC,eAAeD,OAAO,MAAM,CAAC,CAACE,OAAS,CAACA,KAAK,MAAM;IACzD,IAAID,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAO;IAGT,OAAOA,aACJ,GAAG,CAAC,CAACC;QACJ,MAAMC,SAASD,KAAK,OAAO,IAAI;QAC/B,OAAO,CAAC,CAAC,EAAEP,4BAA4BO,MAAM,GAAG,EAAEC,QAAQ;IAC5D,GACC,IAAI,CAAC;AACV;AAEA,eAAeC,yBACbf,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMqB,QAAQjB,AAAoC,UAApCA,aAAa,MAAM,CAAC,WAAW;QAC7C,MAAMC,SAAS,MAAOgB,CAAAA,QAClBC,AAAAA,IAAAA,iDAAAA,eAAAA,AAAAA,EACE;YACE,OAAO;gBAAE,OAAO/B;YAAoB;YACpC,WAAW;gBACT,QAAQ;oBACN,MAAM;oBACN,cAAc,CAAC,iBAAiB,EAAEA,oBAAoB,CAAC,CAAC;gBAC1D;YACF;QACF,GACAa,gBAEFmB,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EACE;YACE;gBACE,MAAM;gBACN,SAAS;YACX;YACA;gBACE,MAAM;gBACN,SAAS,CAAC,eAAe,EAAEhC,qBAAqB;YAClD;SACD,EACDa,aAAY;QAElB,MAAMoB,UAAUnB,OAAO,OAAO,CAAC,IAAI;QACnC,MAAMoB,SAASJ,QACXK,0BAAAA,uBAAAA,CAAAA,KAA6B,CAACC,KAAK,KAAK,CAACH,UAAU,OAAO,CAAC,MAAM,GACjEI;QACJ,MAAMC,SAASR,QACXI,QAAQ,SAAS,UAAUA,OAAO,IAAI,IAAI,MAC1CD,QAAQ,QAAQ,CAACjC;QACrB,OAAOW,iBAAiB,QAAQE,cAAc;YAC5CyB;YACA,YAAY7B,KAAK,GAAG,KAAKoB;YACzB,SAASS,SAAS,KAAK,CAAC,qBAAqB,EAAEL,SAAS;QAC1D;IACF,EAAE,OAAOM,OAAO;QACd,OAAO5B,iBAAiB,QAAQE,cAAc;YAC5C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASU,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeG,2BACb7B,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,IAAII,AAAoC,UAApCA,aAAa,MAAM,CAAC,WAAW,EACjC,MAAMkB,AAAAA,IAAAA,iDAAAA,eAAAA,AAAAA,EACJ;YACE,OAAO,CAAC;YACR,QAAQ;gBAAE,YAAYvB,oCAAAA,0BAA0BA;YAAC;YACjD,WAAW;gBACT,QAAQ;oBACN,MAAM;oBACN,cAAc;gBAChB;YACF;QACF,GACAK;aAGF,MAAMmB,AAAAA,IAAAA,iDAAAA,gBAAAA,AAAAA,EACJ;YACE;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM;oBACR;oBACA;wBACE,MAAM;wBACN,KAAKxB,oCAAAA,0BAA0BA;oBACjC;iBACD;YACH;SACD,EACDK;QAGJ,OAAOF,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAAS;QACX;IACF,EAAE,OAAOU,OAAO;QACd,OAAO5B,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASU,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeI,6BACb9B,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMmC,UAAUtC;QAChB,MAAMuC,UAAU,IAAIC,CAAAA,kBAAAA,EAAQF;QAC5B,MAAMG,eAAe,MAAMF,QAAQ,MAAM,CACvC;YAAE,QAAQ;QAA0B,GACpC,CAAC,GACDhC;QAEF,MAAMmC,SAASD,aAAa,OAAO,EAAE;QACrC,MAAMT,SAASlC,eAAe4C;QAC9B,OAAOrC,iBAAiB,YAAYE,cAAc;YAChDyB;YACA,YAAY7B,KAAK,GAAG,KAAKoB;YACzB,SAASS,SACL,KACA,CAAC,uBAAuB,EAAEF,KAAK,SAAS,CAAC;gBACvCY;YACF,IAAI;QACV;IACF,EAAE,OAAOT,OAAO;QACd,OAAO5B,iBAAiB,YAAYE,cAAc;YAChD,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASU,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEO,eAAeU,oBACpBlC,MAA8B;IAE9B,MAAMmC,WAAW,OACfjC,aACAG;QAEA,MAAM+B,UAAUnC,8BAA8BC;QAC9C,IAAImC,eAAe;QACnB,MAAMC,YAAY,IAAIC;QACtBH,QAAQ,SAAS,GAAG,CAACI;YACnBH,gBAAgB;YAChB,IAAIG,UAAUF,UAAU,GAAG,CAACE;QAC9B;QACA,MAAMzC,SAAS,MAAMM,MAAM+B;QAC3B,OAAO;YAAE,GAAGrC,MAAM;YAAEsC;YAAc,WAAW;mBAAIC;aAAU;QAAC;IAC9D;IACA,MAAM7B,SAAS,MAAMgC,QAAQ,GAAG,CAAC;QAC/BN,SAASnC,OAAO,mBAAmB,EAAEa;QACrCsB,SAASnC,OAAO,kBAAkB,EAAE2B;QACpCQ,SACEnC,OAAO,iBAAiB,IAAIA,OAAO,kBAAkB,EACrD4B;QAEFO,SAASnC,OAAO,kBAAkB,EAAEa;KACrC;IAED,MAAMU,SAASd,OAAO,KAAK,CAAC,CAACE,OAASA,KAAK,MAAM;IACjD,OAAO;QACLF;QACA,cAAcA,OAAO,MAAM,CACzB,CAACiC,OAAOrC,QAAUqC,QAAQrC,MAAM,YAAY,EAC5C;QAEFkB;QACA,SAASA,SAASD,SAAYd,gCAAgCC;IAChE;AACF"}

package/dist/lib/ai-model/index.js

@@ -32,6 +32,7 @@ __webpack_require__.d(__webpack_exports__, {
     runConnectivityTest: ()=>external_connectivity_index_js_namespaceObject.runConnectivityTest,
     parseActionParam: ()=>external_common_js_namespaceObject.parseActionParam,
     createRecorderMarkdownReplayPrompt: ()=>recorder_generation_index_js_namespaceObject.createRecorderMarkdownReplayPrompt,
+    parseSiwcCallback: ()=>siwc_index_js_namespaceObject.parseSiwcCallback,
     convertRecordLogIntoMarkdown: ()=>recorder_generation_index_js_namespaceObject.convertRecordLogIntoMarkdown,
     AiExtractElementInfo: ()=>insight_index_js_namespaceObject.AiExtractElementInfo,
     callAIWithStringResponse: ()=>index_js_namespaceObject.callAIWithStringResponse,
@@ -45,25 +46,35 @@ __webpack_require__.d(__webpack_exports__, {
     PointSchema: ()=>external_common_js_namespaceObject.PointSchema,
     AIResponseParseError: ()=>index_js_namespaceObject.AIResponseParseError,
     RectSchema: ()=>external_common_js_namespaceObject.RectSchema,
+    callGenerativeAI: ()=>index_js_namespaceObject.callGenerativeAI,
+    refreshSiwcCredentials: ()=>siwc_index_js_namespaceObject.refreshSiwcCredentials,
+    tokenResponseError: ()=>errors_js_namespaceObject.tokenResponseError,
     standardPlan: ()=>planning_index_js_namespaceObject.standardPlan,
     generateRecorderYamlTest: ()=>recorder_generation_index_js_namespaceObject.generateRecorderYamlTest,
-    callAI: ()=>index_js_namespaceObject.callAI,
     AiLocateElement: ()=>grounding_index_js_namespaceObject.AiLocateElement,
     getModelRuntime: ()=>external_models_index_js_namespaceObject.getModelRuntime,
+    SIWC_AUTH_STORAGE_KEY: ()=>siwc_index_js_namespaceObject.SIWC_AUTH_STORAGE_KEY,
+    exchangeSiwcCode: ()=>siwc_index_js_namespaceObject.exchangeSiwcCode,
     generateRecorderMarkdownReplay: ()=>recorder_generation_index_js_namespaceObject.generateRecorderMarkdownReplay,
+    createSiwcAuthorization: ()=>siwc_index_js_namespaceObject.createSiwcAuthorization,
     findAllMidsceneLocatorField: ()=>external_common_js_namespaceObject.findAllMidsceneLocatorField,
+    callSystemOneAI: ()=>index_js_namespaceObject.callSystemOneAI,
     generatePlaywrightTestStream: ()=>recorder_generation_index_js_namespaceObject.generatePlaywrightTestStream,
+    withSiwcStage: ()=>errors_js_namespaceObject.withSiwcStage,
     AiLocateSection: ()=>grounding_index_js_namespaceObject.AiLocateSection,
-    TMultimodalPromptSchema: ()=>external_common_js_namespaceObject.TMultimodalPromptSchema
+    TMultimodalPromptSchema: ()=>external_common_js_namespaceObject.TMultimodalPromptSchema,
+    SiwcError: ()=>errors_js_namespaceObject.SiwcError
 });
 const index_js_namespaceObject = require("./service-caller/index.js");
 const external_models_index_js_namespaceObject = require("./models/index.js");
+const siwc_index_js_namespaceObject = require("./service-caller/openai/siwc/index.js");
 const external_connectivity_index_js_namespaceObject = require("./connectivity/index.js");
 const recorder_generation_index_js_namespaceObject = require("./workflows/recorder-generation/index.js");
 const grounding_index_js_namespaceObject = require("./workflows/grounding/index.js");
 const insight_index_js_namespaceObject = require("./workflows/insight/index.js");
 const planning_index_js_namespaceObject = require("./workflows/planning/index.js");
 const external_common_js_namespaceObject = require("../common.js");
+const errors_js_namespaceObject = require("./service-caller/openai/siwc/errors.js");
 exports.AIResponseParseError = __webpack_exports__.AIResponseParseError;
 exports.AiExtractElementInfo = __webpack_exports__.AiExtractElementInfo;
 exports.AiJudgeOrderSensitive = __webpack_exports__.AiJudgeOrderSensitive;
@@ -72,15 +83,20 @@ exports.AiLocateSection = __webpack_exports__.AiLocateSection;
 exports.ConversationHistory = __webpack_exports__.ConversationHistory;
 exports.PointSchema = __webpack_exports__.PointSchema;
 exports.RectSchema = __webpack_exports__.RectSchema;
+exports.SIWC_AUTH_STORAGE_KEY = __webpack_exports__.SIWC_AUTH_STORAGE_KEY;
+exports.SiwcError = __webpack_exports__.SiwcError;
 exports.SizeSchema = __webpack_exports__.SizeSchema;
 exports.TMultimodalPromptSchema = __webpack_exports__.TMultimodalPromptSchema;
 exports.TUserPromptSchema = __webpack_exports__.TUserPromptSchema;
-exports.callAI = __webpack_exports__.callAI;
 exports.callAIWithObjectResponse = __webpack_exports__.callAIWithObjectResponse;
 exports.callAIWithStringResponse = __webpack_exports__.callAIWithStringResponse;
+exports.callGenerativeAI = __webpack_exports__.callGenerativeAI;
+exports.callSystemOneAI = __webpack_exports__.callSystemOneAI;
 exports.convertRecordLogIntoMarkdown = __webpack_exports__.convertRecordLogIntoMarkdown;
 exports.createRecorderMarkdownReplayPrompt = __webpack_exports__.createRecorderMarkdownReplayPrompt;
+exports.createSiwcAuthorization = __webpack_exports__.createSiwcAuthorization;
 exports.dumpActionParam = __webpack_exports__.dumpActionParam;
+exports.exchangeSiwcCode = __webpack_exports__.exchangeSiwcCode;
 exports.findAllMidsceneLocatorField = __webpack_exports__.findAllMidsceneLocatorField;
 exports.generatePlaywrightTest = __webpack_exports__.generatePlaywrightTest;
 exports.generatePlaywrightTestStream = __webpack_exports__.generatePlaywrightTestStream;
@@ -93,8 +109,12 @@ exports.generateYamlTestStream = __webpack_exports__.generateYamlTestStream;
 exports.getMidsceneLocationSchema = __webpack_exports__.getMidsceneLocationSchema;
 exports.getModelRuntime = __webpack_exports__.getModelRuntime;
 exports.parseActionParam = __webpack_exports__.parseActionParam;
+exports.parseSiwcCallback = __webpack_exports__.parseSiwcCallback;
+exports.refreshSiwcCredentials = __webpack_exports__.refreshSiwcCredentials;
 exports.runConnectivityTest = __webpack_exports__.runConnectivityTest;
 exports.standardPlan = __webpack_exports__.standardPlan;
+exports.tokenResponseError = __webpack_exports__.tokenResponseError;
+exports.withSiwcStage = __webpack_exports__.withSiwcStage;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
     "AIResponseParseError",
     "AiExtractElementInfo",
@@ -104,15 +124,20 @@ for(var __rspack_i in __webpack_exports__)if (-1 === [
     "ConversationHistory",
     "PointSchema",
     "RectSchema",
+    "SIWC_AUTH_STORAGE_KEY",
+    "SiwcError",
     "SizeSchema",
     "TMultimodalPromptSchema",
     "TUserPromptSchema",
-    "callAI",
     "callAIWithObjectResponse",
     "callAIWithStringResponse",
+    "callGenerativeAI",
+    "callSystemOneAI",
     "convertRecordLogIntoMarkdown",
     "createRecorderMarkdownReplayPrompt",
+    "createSiwcAuthorization",
     "dumpActionParam",
+    "exchangeSiwcCode",
     "findAllMidsceneLocatorField",
     "generatePlaywrightTest",
     "generatePlaywrightTestStream",
@@ -125,8 +150,12 @@ for(var __rspack_i in __webpack_exports__)if (-1 === [
     "getMidsceneLocationSchema",
     "getModelRuntime",
     "parseActionParam",
+    "parseSiwcCallback",
+    "refreshSiwcCredentials",
     "runConnectivityTest",
-    "standardPlan"
+    "standardPlan",
+    "tokenResponseError",
+    "withSiwcStage"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true

package/dist/lib/ai-model/model-adapter/resolve.js

@@ -56,7 +56,8 @@ function resolveImagePreprocess(imagePreprocess) {
 }
 class ResolvedModelAdapter {
     constructor(config, modelFamily){
-        _define_property(this, "supportedTasks", void 0);
+        _define_property(this, "modelType", void 0);
+        _define_property(this, "supportedTaskTypes", void 0);
         _define_property(this, "supportedProtocols", void 0);
         _define_property(this, "jsonParser", void 0);
         _define_property(this, "chatCompletion", void 0);
@@ -67,7 +68,8 @@ class ResolvedModelAdapter {
         _define_property(this, "insight", void 0);
         _define_property(this, "planning", void 0);
         _define_property(this, "locate", void 0);
-        this.supportedTasks = config.supportedTasks;
+        this.modelType = config.modelType ?? 'generative';
+        this.supportedTaskTypes = config.supportedTaskTypes;
         this.supportedProtocols = config.supportedProtocols ?? [
             'openai-chat'
         ];

package/dist/lib/ai-model/model-adapter/resolve.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/resolve.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { parseModelResponseJson } from '../shared/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { buildDefaultCodexAppServerParams } from './codex-app-server';\nimport { resolveInsight } from './insight';\nimport type { InsightAdapter } from './insight-protocol';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport { resolveResponses } from './responses';\nimport type {\n  BuildCodexAppServerParams,\n  ChatCompletionAdapter,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n  ResolveImageDetail,\n  ResponsesAdapter,\n} from './types';\n\nconst defaultImageDetail: ResolveImageDetail = ({ imageDetail }) =>\n  imageDetail ?? 'high';\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return parseModelResponseJson;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly supportedTasks: ModelAdapter['supportedTasks'];\n  readonly supportedProtocols: ModelAdapter['supportedProtocols'];\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly resolveImageDetail: ResolveImageDetail;\n  readonly responses: ResponsesAdapter;\n  readonly buildCodexAppServerParams: BuildCodexAppServerParams;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly insight: InsightAdapter;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.supportedTasks = config.supportedTasks;\n    this.supportedProtocols = config.supportedProtocols ?? ['openai-chat'];\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.resolveImageDetail = config.resolveImageDetail ?? defaultImageDetail;\n    this.responses = resolveResponses(config.responses);\n    this.buildCodexAppServerParams =\n      config.buildCodexAppServerParams ?? buildDefaultCodexAppServerParams;\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    this.insight = resolveInsight(config.insight, {\n      jsonParser: this.jsonParser,\n    });\n    const customPlanner =\n      config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n    const resolvedCustomPlanner = customPlanner\n      ? resolveCustomPlanningDefinition(customPlanner)\n      : undefined;\n    this.locate = resolveLocate(config.locate, resolvedCustomPlanner, {\n      jsonParser: this.jsonParser,\n    });\n    this.planning = resolvePlanning(\n      config.planning,\n      resolvedCustomPlanner,\n      { jsonParser: this.jsonParser },\n      this.locate.kind === 'standard'\n        ? this.locate.element.resultCodec\n        : undefined,\n    );\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultImageDetail","imageDetail","resolveJsonParser","jsonParser","parseModelResponseJson","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","resolveResponses","buildDefaultCodexAppServerParams","resolveInsight","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolveLocate","resolvePlanning"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;ACeA,MAAMI,qBAAyC,CAAC,EAAEC,WAAW,EAAE,GAC7DA,eAAe;AAEjB,SAASC,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC,wBAAAA,sBAAsBA;IAG/B,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAaX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QAZjE,uBAAS,kBAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,aAAT;QACA,uBAAS,6BAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,WAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,cAAc,GAAGD,OAAO,cAAc;QAC3C,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAI;YAAC;SAAc;QACtE,IAAI,CAAC,UAAU,GAAGP,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,AAAAA,IAAAA,4CAAAA,qBAAAA,AAAAA,EAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAIT;QACvD,IAAI,CAAC,SAAS,GAAGY,AAAAA,IAAAA,sCAAAA,gBAAAA,AAAAA,EAAiBH,OAAO,SAAS;QAClD,IAAI,CAAC,yBAAyB,GAC5BA,OAAO,yBAAyB,IAAII,6CAAAA,gCAAgCA;QACtE,IAAI,CAAC,eAAe,GAAGP,uBAAuBG,OAAO,eAAe;QACpE,IAAI,CAAC,OAAO,GAAGK,AAAAA,IAAAA,oCAAAA,cAAAA,AAAAA,EAAeL,OAAO,OAAO,EAAE;YAC5C,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,MAAMM,gBACJN,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGO;QACjE,MAAMC,wBAAwBF,gBAC1BG,AAAAA,IAAAA,qCAAAA,+BAAAA,AAAAA,EAAgCH,iBAChCC;QACJ,IAAI,CAAC,MAAM,GAAGG,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAcV,OAAO,MAAM,EAAEQ,uBAAuB;YAChE,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,IAAI,CAAC,QAAQ,GAAGG,AAAAA,IAAAA,qCAAAA,eAAAA,AAAAA,EACdX,OAAO,QAAQ,EACfQ,uBACA;YAAE,YAAY,IAAI,CAAC,UAAU;QAAC,GAC9B,AAAqB,eAArB,IAAI,CAAC,MAAM,CAAC,IAAI,GACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,GAC/BD;IAER;AACF"}
+{"version":3,"file":"ai-model/model-adapter/resolve.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { parseModelResponseJson } from '../shared/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { buildDefaultCodexAppServerParams } from './codex-app-server';\nimport { resolveInsight } from './insight';\nimport type { InsightAdapter } from './insight-protocol';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport { resolveResponses } from './responses';\nimport type {\n  BuildCodexAppServerParams,\n  ChatCompletionAdapter,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n  ResolveImageDetail,\n  ResponsesAdapter,\n} from './types';\n\nconst defaultImageDetail: ResolveImageDetail = ({ imageDetail }) =>\n  imageDetail ?? 'high';\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return parseModelResponseJson;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly modelType: ModelAdapter['modelType'];\n  readonly supportedTaskTypes: ModelAdapter['supportedTaskTypes'];\n  readonly supportedProtocols: ModelAdapter['supportedProtocols'];\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly resolveImageDetail: ResolveImageDetail;\n  readonly responses: ResponsesAdapter;\n  readonly buildCodexAppServerParams: BuildCodexAppServerParams;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly insight: InsightAdapter;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.modelType = config.modelType ?? 'generative';\n    this.supportedTaskTypes = config.supportedTaskTypes;\n    this.supportedProtocols = config.supportedProtocols ?? ['openai-chat'];\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.resolveImageDetail = config.resolveImageDetail ?? defaultImageDetail;\n    this.responses = resolveResponses(config.responses);\n    this.buildCodexAppServerParams =\n      config.buildCodexAppServerParams ?? buildDefaultCodexAppServerParams;\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    this.insight = resolveInsight(config.insight, {\n      jsonParser: this.jsonParser,\n    });\n    const customPlanner =\n      config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n    const resolvedCustomPlanner = customPlanner\n      ? resolveCustomPlanningDefinition(customPlanner)\n      : undefined;\n    this.locate = resolveLocate(config.locate, resolvedCustomPlanner, {\n      jsonParser: this.jsonParser,\n    });\n    this.planning = resolvePlanning(\n      config.planning,\n      resolvedCustomPlanner,\n      { jsonParser: this.jsonParser },\n      this.locate.kind === 'standard'\n        ? this.locate.element.resultCodec\n        : undefined,\n    );\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultImageDetail","imageDetail","resolveJsonParser","jsonParser","parseModelResponseJson","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","resolveResponses","buildDefaultCodexAppServerParams","resolveInsight","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolveLocate","resolvePlanning"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;ACeA,MAAMI,qBAAyC,CAAC,EAAEC,WAAW,EAAE,GAC7DA,eAAe;AAEjB,SAASC,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC,wBAAAA,sBAAsBA;IAG/B,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAcX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QAbjE,uBAAS,aAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,aAAT;QACA,uBAAS,6BAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,WAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,SAAS,GAAGD,OAAO,SAAS,IAAI;QACrC,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB;QACnD,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAI;YAAC;SAAc;QACtE,IAAI,CAAC,UAAU,GAAGP,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,AAAAA,IAAAA,4CAAAA,qBAAAA,AAAAA,EAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAIT;QACvD,IAAI,CAAC,SAAS,GAAGY,AAAAA,IAAAA,sCAAAA,gBAAAA,AAAAA,EAAiBH,OAAO,SAAS;QAClD,IAAI,CAAC,yBAAyB,GAC5BA,OAAO,yBAAyB,IAAII,6CAAAA,gCAAgCA;QACtE,IAAI,CAAC,eAAe,GAAGP,uBAAuBG,OAAO,eAAe;QACpE,IAAI,CAAC,OAAO,GAAGK,AAAAA,IAAAA,oCAAAA,cAAAA,AAAAA,EAAeL,OAAO,OAAO,EAAE;YAC5C,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,MAAMM,gBACJN,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGO;QACjE,MAAMC,wBAAwBF,gBAC1BG,AAAAA,IAAAA,qCAAAA,+BAAAA,AAAAA,EAAgCH,iBAChCC;QACJ,IAAI,CAAC,MAAM,GAAGG,AAAAA,IAAAA,mCAAAA,aAAAA,AAAAA,EAAcV,OAAO,MAAM,EAAEQ,uBAAuB;YAChE,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,IAAI,CAAC,QAAQ,GAAGG,AAAAA,IAAAA,qCAAAA,eAAAA,AAAAA,EACdX,OAAO,QAAQ,EACfQ,uBACA;YAAE,YAAY,IAAI,CAAC,UAAU;QAAC,GAC9B,AAAqB,eAArB,IAAI,CAAC,MAAM,CAAC,IAAI,GACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,GAC/BD;IAER;AACF"}

package/dist/lib/ai-model/models/jev/adapter.js

@@ -26,117 +26,13 @@ __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
     jevAdapters: ()=>jevAdapters
 });
-const external_common_js_namespaceObject = require("../../../common.js");
-const utils_namespaceObject = require("@midscene/shared/utils");
-const index_js_namespaceObject = require("../../service-caller/index.js");
-const system_one_js_namespaceObject = require("../../service-caller/system-one.js");
-const multimodal_prompt_js_namespaceObject = require("../../shared/multimodal-prompt.js");
-const external_image_js_namespaceObject = require("./image.js");
+const external_boolean_js_namespaceObject = require("./boolean.js");
+const external_locate_js_namespaceObject = require("./locate.js");
 const external_planning_js_namespaceObject = require("./planning.js");
-async function prepareReferences(prompt) {
-    const { referenceImages } = await (0, multimodal_prompt_js_namespaceObject.prepareUserPrompt)(prompt);
-    const prepared = await Promise.all(referenceImages.map(async (image, index)=>({
-            key: `reference_${index}`,
-            name: image.name,
-            image: await (0, external_image_js_namespaceObject.prepareJevImage)(image.url)
-        })));
-    return {
-        references: prepared.map(({ key, name })=>({
-                image: `\`${key}\``,
-                name
-            })),
-        images: Object.fromEntries(prepared.map(({ key, image })=>[
-                key,
-                image.imageBase64
-            ]))
-    };
-}
-const locate = async ({ targetElementDescription, locateImage, options })=>{
-    const image = await (0, external_image_js_namespaceObject.prepareJevImage)(locateImage.imageBase64);
-    const references = await prepareReferences(targetElementDescription);
-    const target = (0, external_common_js_namespaceObject.userPromptToString)(targetElementDescription);
-    const response = await (0, index_js_namespaceObject.callAI)([], options.modelRuntime, {
-        abortSignal: options.abortSignal,
-        systemOneRequest: {
-            state: {
-                target,
-                references: references.references
-            },
-            images: {
-                screenshot: image.imageBase64,
-                ...references.images
-            },
-            questions: {
-                present: {
-                    type: 'noul',
-                    instructions: 'Is the target described in `target` present in `screenshot`? Reference images, if any, describe the target; they are not the screenshot.'
-                },
-                target: {
-                    type: 'box',
-                    instructions: 'The single target described in `target`, in `screenshot`.'
-                }
-            }
-        }
-    });
-    const { answers } = system_one_js_namespaceObject.systemOneResponseSchema.parse(JSON.parse(response.content));
-    (0, utils_namespaceObject.assert)('noul' === answers.present.type && 'box' === answers.target.type, 'Invalid Jev locate answers');
-    return {
-        rawResponse: response.content,
-        usage: response.usage,
-        ...answers.present.noul >= 0.5 ? {
-            locatedPixelResult: (0, external_image_js_namespaceObject.mapJevBox)(answers.target.box, image, locateImage)
-        } : {
-            errors: [
-                'The target is not present in the screenshot'
-            ]
-        }
-    };
-};
-const adapter_boolean = async (options)=>{
-    const references = await prepareReferences({
-        prompt: '',
-        ...options.multimodalPrompt
-    });
-    const frames = options.extractOption?.screenshotIncluded === false ? [] : options.context.screenshotSequence?.length ? options.context.screenshotSequence : [
-        options.context.screenshot
-    ];
-    const screenshots = await Promise.all(frames.map((frame)=>(0, external_image_js_namespaceObject.prepareJevImage)(frame.base64)));
-    const result = await (0, index_js_namespaceObject.callAI)([], options.modelRuntime, {
-        abortSignal: options.abortSignal,
-        systemOneRequest: {
-            state: {
-                statement: options.dataQuery,
-                context: options.extractOption?.context,
-                pageDescription: options.pageDescription,
-                frames: screenshots.map((_, index)=>`\`frame_${index}\``),
-                references: references.references
-            },
-            images: {
-                ...Object.fromEntries(screenshots.map((image, index)=>[
-                        `frame_${index}`,
-                        image.imageBase64
-                    ])),
-                ...references.images
-            },
-            questions: {
-                result: {
-                    type: 'noul',
-                    instructions: "Is `statement` true? Screenshots in `frames` are ordered earliest to latest. For current state use the latest frame; for events use the relevant frames. Reference images are supporting context, not screenshots, unless the statement requests comparison. Use the provided context and page description when present."
-                }
-            }
-        }
-    });
-    const answer = system_one_js_namespaceObject.systemOneResponseSchema.parse(JSON.parse(result.content)).answers.result;
-    (0, utils_namespaceObject.assert)('noul' === answer.type, 'Invalid Jev boolean answer');
-    return {
-        value: answer.noul >= 0.5,
-        rawResponse: result.content,
-        usage: result.usage
-    };
-};
 const jevAdapters = {
     jev: {
-        supportedTasks: [
+        modelType: 'system-one',
+        supportedTaskTypes: [
             'locate',
             'boolean',
             'planning'
@@ -144,10 +40,10 @@ const jevAdapters = {
         supportedProtocols: [],
         locate: {
             kind: 'custom',
-            locateFn: locate
+            locateFn: external_locate_js_namespaceObject.jevLocate
         },
         insight: {
-            booleanFn: adapter_boolean
+            booleanFn: external_boolean_js_namespaceObject.jevBoolean
         },
         planning: {
             kind: 'custom',

package/dist/lib/ai-model/models/jev/adapter.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/jev/adapter.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/jev/adapter.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { userPromptToString } from '@/common';\nimport type { TModelFamily } from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\nimport type { ModelAdapterDefinition } from '../../model-adapter/types';\nimport { callAI } from '../../service-caller';\nimport { systemOneResponseSchema } from '../../service-caller/system-one';\nimport { prepareUserPrompt } from '../../shared/multimodal-prompt';\nimport type { LocateFn } from '../../workflows/grounding/types';\nimport type { BooleanFn } from '../../workflows/insight/boolean';\nimport { mapJevBox, prepareJevImage } from './image';\nimport { jevPlan } from './planning';\n\nasync function prepareReferences(\n  prompt: Parameters<typeof prepareUserPrompt>[0],\n) {\n  const { referenceImages } = await prepareUserPrompt(prompt);\n  const prepared = await Promise.all(\n    referenceImages.map(async (image, index) => ({\n      key: `reference_${index}`,\n      name: image.name,\n      image: await prepareJevImage(image.url),\n    })),\n  );\n  return {\n    references: prepared.map(({ key, name }) => ({\n      image: `\\`${key}\\``,\n      name,\n    })),\n    images: Object.fromEntries(\n      prepared.map(({ key, image }) => [key, image.imageBase64]),\n    ),\n  };\n}\n\nconst locate: LocateFn = async ({\n  targetElementDescription,\n  locateImage,\n  options,\n}) => {\n  const image = await prepareJevImage(locateImage.imageBase64);\n  const references = await prepareReferences(targetElementDescription);\n  const target = userPromptToString(targetElementDescription);\n  const response = await callAI([], options.modelRuntime, {\n    abortSignal: options.abortSignal,\n    systemOneRequest: {\n      state: { target, references: references.references },\n      images: { screenshot: image.imageBase64, ...references.images },\n      questions: {\n        present: {\n          type: 'noul',\n          instructions:\n            'Is the target described in `target` present in `screenshot`? Reference images, if any, describe the target; they are not the screenshot.',\n        },\n        target: {\n          type: 'box',\n          instructions:\n            'The single target described in `target`, in `screenshot`.',\n        },\n      },\n    },\n  });\n  const { answers } = systemOneResponseSchema.parse(\n    JSON.parse(response.content),\n  );\n  assert(\n    answers.present.type === 'noul' && answers.target.type === 'box',\n    'Invalid Jev locate answers',\n  );\n  return {\n    rawResponse: response.content,\n    usage: response.usage,\n    ...(answers.present.noul >= 0.5\n      ? {\n          locatedPixelResult: mapJevBox(answers.target.box, image, locateImage),\n        }\n      : { errors: ['The target is not present in the screenshot'] }),\n  };\n};\n\nconst boolean: BooleanFn = async (options) => {\n  const references = await prepareReferences({\n    prompt: '',\n    ...options.multimodalPrompt,\n  });\n  const frames =\n    options.extractOption?.screenshotIncluded === false\n      ? []\n      : options.context.screenshotSequence?.length\n        ? options.context.screenshotSequence\n        : [options.context.screenshot];\n  const screenshots = await Promise.all(\n    frames.map((frame) => prepareJevImage(frame.base64)),\n  );\n  const result = await callAI([], options.modelRuntime, {\n    abortSignal: options.abortSignal,\n    systemOneRequest: {\n      state: {\n        statement: options.dataQuery,\n        context: options.extractOption?.context,\n        pageDescription: options.pageDescription,\n        frames: screenshots.map((_, index) => `\\`frame_${index}\\``),\n        references: references.references,\n      },\n      images: {\n        ...Object.fromEntries(\n          screenshots.map((image, index) => [\n            `frame_${index}`,\n            image.imageBase64,\n          ]),\n        ),\n        ...references.images,\n      },\n      questions: {\n        result: {\n          type: 'noul',\n          instructions:\n            'Is `statement` true? Screenshots in `frames` are ordered earliest to latest. For current state use the latest frame; for events use the relevant frames. Reference images are supporting context, not screenshots, unless the statement requests comparison. Use the provided context and page description when present.',\n        },\n      },\n    },\n  });\n  const answer = systemOneResponseSchema.parse(JSON.parse(result.content))\n    .answers.result;\n  assert(answer.type === 'noul', 'Invalid Jev boolean answer');\n  return {\n    value: answer.noul >= 0.5,\n    rawResponse: result.content,\n    usage: result.usage,\n  };\n};\n\nexport const jevAdapters = {\n  jev: {\n    supportedTasks: ['locate', 'boolean', 'planning'],\n    supportedProtocols: [],\n    locate: { kind: 'custom', locateFn: locate },\n    insight: { booleanFn: boolean },\n    planning: {\n      kind: 'custom',\n      planFn: jevPlan,\n      cacheEnabled: false,\n      supportsActionDeepLocate: false,\n      defaultReplanningCycleLimit: 20,\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'jev'>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","prepareReferences","prompt","referenceImages","prepareUserPrompt","prepared","Promise","image","index","prepareJevImage","name","locate","targetElementDescription","locateImage","options","references","target","userPromptToString","response","callAI","answers","systemOneResponseSchema","JSON","assert","mapJevBox","boolean","frames","screenshots","frame","result","_","answer","jevAdapters","jevPlan"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;ACMA,eAAeI,kBACbC,MAA+C;IAE/C,MAAM,EAAEC,eAAe,EAAE,GAAG,MAAMC,AAAAA,IAAAA,qCAAAA,iBAAAA,AAAAA,EAAkBF;IACpD,MAAMG,WAAW,MAAMC,QAAQ,GAAG,CAChCH,gBAAgB,GAAG,CAAC,OAAOI,OAAOC,QAAW;YAC3C,KAAK,CAAC,UAAU,EAAEA,OAAO;YACzB,MAAMD,MAAM,IAAI;YAChB,OAAO,MAAME,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBF,MAAM,GAAG;QACxC;IAEF,OAAO;QACL,YAAYF,SAAS,GAAG,CAAC,CAAC,EAAET,GAAG,EAAEc,IAAI,EAAE,GAAM;gBAC3C,OAAO,CAAC,EAAE,EAAEd,IAAI,EAAE,CAAC;gBACnBc;YACF;QACA,QAAQb,OAAO,WAAW,CACxBQ,SAAS,GAAG,CAAC,CAAC,EAAET,GAAG,EAAEW,KAAK,EAAE,GAAK;gBAACX;gBAAKW,MAAM,WAAW;aAAC;IAE7D;AACF;AAEA,MAAMI,SAAmB,OAAO,EAC9BC,wBAAwB,EACxBC,WAAW,EACXC,OAAO,EACR;IACC,MAAMP,QAAQ,MAAME,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBI,YAAY,WAAW;IAC3D,MAAME,aAAa,MAAMd,kBAAkBW;IAC3C,MAAMI,SAASC,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBL;IAClC,MAAMM,WAAW,MAAMC,AAAAA,IAAAA,yBAAAA,MAAAA,AAAAA,EAAO,EAAE,EAAEL,QAAQ,YAAY,EAAE;QACtD,aAAaA,QAAQ,WAAW;QAChC,kBAAkB;YAChB,OAAO;gBAAEE;gBAAQ,YAAYD,WAAW,UAAU;YAAC;YACnD,QAAQ;gBAAE,YAAYR,MAAM,WAAW;gBAAE,GAAGQ,WAAW,MAAM;YAAC;YAC9D,WAAW;gBACT,SAAS;oBACP,MAAM;oBACN,cACE;gBACJ;gBACA,QAAQ;oBACN,MAAM;oBACN,cACE;gBACJ;YACF;QACF;IACF;IACA,MAAM,EAAEK,OAAO,EAAE,GAAGC,8BAAAA,uBAAAA,CAAAA,KAA6B,CAC/CC,KAAK,KAAK,CAACJ,SAAS,OAAO;IAE7BK,IAAAA,sBAAAA,MAAAA,AAAAA,EACEH,AAAyB,WAAzBA,QAAQ,OAAO,CAAC,IAAI,IAAeA,AAAwB,UAAxBA,QAAQ,MAAM,CAAC,IAAI,EACtD;IAEF,OAAO;QACL,aAAaF,SAAS,OAAO;QAC7B,OAAOA,SAAS,KAAK;QACrB,GAAIE,QAAQ,OAAO,CAAC,IAAI,IAAI,MACxB;YACE,oBAAoBI,AAAAA,IAAAA,kCAAAA,SAAAA,AAAAA,EAAUJ,QAAQ,MAAM,CAAC,GAAG,EAAEb,OAAOM;QAC3D,IACA;YAAE,QAAQ;gBAAC;aAA8C;QAAC,CAAC;IACjE;AACF;AAEA,MAAMY,kBAAqB,OAAOX;IAChC,MAAMC,aAAa,MAAMd,kBAAkB;QACzC,QAAQ;QACR,GAAGa,QAAQ,gBAAgB;IAC7B;IACA,MAAMY,SACJZ,QAAQ,aAAa,EAAE,uBAAuB,QAC1C,EAAE,GACFA,QAAQ,OAAO,CAAC,kBAAkB,EAAE,SAClCA,QAAQ,OAAO,CAAC,kBAAkB,GAClC;QAACA,QAAQ,OAAO,CAAC,UAAU;KAAC;IACpC,MAAMa,cAAc,MAAMrB,QAAQ,GAAG,CACnCoB,OAAO,GAAG,CAAC,CAACE,QAAUnB,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBmB,MAAM,MAAM;IAEpD,MAAMC,SAAS,MAAMV,AAAAA,IAAAA,yBAAAA,MAAAA,AAAAA,EAAO,EAAE,EAAEL,QAAQ,YAAY,EAAE;QACpD,aAAaA,QAAQ,WAAW;QAChC,kBAAkB;YAChB,OAAO;gBACL,WAAWA,QAAQ,SAAS;gBAC5B,SAASA,QAAQ,aAAa,EAAE;gBAChC,iBAAiBA,QAAQ,eAAe;gBACxC,QAAQa,YAAY,GAAG,CAAC,CAACG,GAAGtB,QAAU,CAAC,QAAQ,EAAEA,MAAM,EAAE,CAAC;gBAC1D,YAAYO,WAAW,UAAU;YACnC;YACA,QAAQ;gBACN,GAAGlB,OAAO,WAAW,CACnB8B,YAAY,GAAG,CAAC,CAACpB,OAAOC,QAAU;wBAChC,CAAC,MAAM,EAAEA,OAAO;wBAChBD,MAAM,WAAW;qBAClB,EACF;gBACD,GAAGQ,WAAW,MAAM;YACtB;YACA,WAAW;gBACT,QAAQ;oBACN,MAAM;oBACN,cACE;gBACJ;YACF;QACF;IACF;IACA,MAAMgB,SAASV,8BAAAA,uBAAAA,CAAAA,KAA6B,CAACC,KAAK,KAAK,CAACO,OAAO,OAAO,GACnE,OAAO,CAAC,MAAM;IACjBN,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOQ,AAAgB,WAAhBA,OAAO,IAAI,EAAa;IAC/B,OAAO;QACL,OAAOA,OAAO,IAAI,IAAI;QACtB,aAAaF,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,MAAMG,cAAc;IACzB,KAAK;QACH,gBAAgB;YAAC;YAAU;YAAW;SAAW;QACjD,oBAAoB,EAAE;QACtB,QAAQ;YAAE,MAAM;YAAU,UAAUrB;QAAO;QAC3C,SAAS;YAAE,WAAWc;QAAQ;QAC9B,UAAU;YACR,MAAM;YACN,QAAQQ,qCAAAA,OAAOA;YACf,cAAc;YACd,0BAA0B;YAC1B,6BAA6B;QAC/B;IACF;AACF"}
+{"version":3,"file":"ai-model/models/jev/adapter.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/jev/adapter.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TModelFamily } from '@midscene/shared/env';\nimport type { ModelAdapterDefinition } from '../../model-adapter/types';\nimport { jevBoolean } from './boolean';\nimport { jevLocate } from './locate';\nimport { jevPlan } from './planning';\n\nexport const jevAdapters = {\n  jev: {\n    modelType: 'system-one',\n    supportedTaskTypes: ['locate', 'boolean', 'planning'],\n    supportedProtocols: [],\n    locate: { kind: 'custom', locateFn: jevLocate },\n    insight: { booleanFn: jevBoolean },\n    planning: {\n      kind: 'custom',\n      planFn: jevPlan,\n      cacheEnabled: false,\n      supportsActionDeepLocate: false,\n      defaultReplanningCycleLimit: 20,\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'jev'>;\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","jevAdapters","jevLocate","jevBoolean","jevPlan"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;ACAO,MAAMI,cAAc;IACzB,KAAK;QACH,WAAW;QACX,oBAAoB;YAAC;YAAU;YAAW;SAAW;QACrD,oBAAoB,EAAE;QACtB,QAAQ;YAAE,MAAM;YAAU,UAAUC,mCAAAA,SAASA;QAAC;QAC9C,SAAS;YAAE,WAAWC,oCAAAA,UAAUA;QAAC;QACjC,UAAU;YACR,MAAM;YACN,QAAQC,qCAAAA,OAAOA;YACf,cAAc;YACd,0BAA0B;YAC1B,6BAA6B;QAC/B;IACF;AACF"}

package/dist/lib/ai-model/models/jev/boolean.js

@@ -0,0 +1,82 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    jevBoolean: ()=>jevBoolean
+});
+const utils_namespaceObject = require("@midscene/shared/utils");
+const index_js_namespaceObject = require("../../service-caller/index.js");
+const schema_js_namespaceObject = require("../../service-caller/system-one/schema.js");
+const external_image_js_namespaceObject = require("./image.js");
+const jevBoolean = async (options)=>{
+    const references = await (0, external_image_js_namespaceObject.prepareReferences)({
+        prompt: '',
+        ...options.multimodalPrompt
+    });
+    const frames = options.extractOption?.screenshotIncluded === false ? [] : options.context.screenshotSequence?.length ? options.context.screenshotSequence : [
+        options.context.screenshot
+    ];
+    const screenshots = await Promise.all(frames.map((frame)=>(0, external_image_js_namespaceObject.prepareJevImage)(frame.base64)));
+    const result = await (0, index_js_namespaceObject.callSystemOneAI)({
+        state: {
+            statement: options.dataQuery,
+            context: options.extractOption?.context,
+            pageDescription: options.pageDescription,
+            frames: screenshots.map((_, index)=>`\`frame_${index}\``),
+            references: references.references
+        },
+        images: {
+            ...Object.fromEntries(screenshots.map((image, index)=>[
+                    `frame_${index}`,
+                    image.imageBase64
+                ])),
+            ...references.images
+        },
+        questions: {
+            result: {
+                type: 'noul',
+                instructions: "Is `statement` true? Screenshots in `frames` are ordered earliest to latest. For current state use the latest frame; for events use the relevant frames. Reference images are supporting context, not screenshots, unless the statement requests comparison. Use the provided context and page description when present."
+            }
+        }
+    }, options.modelRuntime, {
+        abortSignal: options.abortSignal
+    });
+    const answer = schema_js_namespaceObject.systemOneResponseSchema.parse(JSON.parse(result.content)).answers.result;
+    (0, utils_namespaceObject.assert)('noul' === answer.type, 'Invalid Jev boolean answer');
+    return {
+        value: answer.noul >= 0.5,
+        rawResponse: result.content,
+        usage: result.usage
+    };
+};
+exports.jevBoolean = __webpack_exports__.jevBoolean;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "jevBoolean"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=boolean.js.map

package/dist/lib/ai-model/models/jev/boolean.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/jev/boolean.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../../src/ai-model/models/jev/boolean.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { assert } from '@midscene/shared/utils';\nimport { callSystemOneAI } from '../../service-caller';\nimport { systemOneResponseSchema } from '../../service-caller/system-one/schema';\nimport type { BooleanFn } from '../../workflows/insight/boolean';\nimport { prepareJevImage, prepareReferences } from './image';\n\nexport const jevBoolean: BooleanFn = async (options) => {\n  const references = await prepareReferences({\n    prompt: '',\n    ...options.multimodalPrompt,\n  });\n  const frames =\n    options.extractOption?.screenshotIncluded === false\n      ? []\n      : options.context.screenshotSequence?.length\n        ? options.context.screenshotSequence\n        : [options.context.screenshot];\n  const screenshots = await Promise.all(\n    frames.map((frame) => prepareJevImage(frame.base64)),\n  );\n  const result = await callSystemOneAI(\n    {\n      state: {\n        statement: options.dataQuery,\n        context: options.extractOption?.context,\n        pageDescription: options.pageDescription,\n        frames: screenshots.map((_, index) => `\\`frame_${index}\\``),\n        references: references.references,\n      },\n      images: {\n        ...Object.fromEntries(\n          screenshots.map((image, index) => [\n            `frame_${index}`,\n            image.imageBase64,\n          ]),\n        ),\n        ...references.images,\n      },\n      questions: {\n        result: {\n          type: 'noul',\n          instructions:\n            'Is `statement` true? Screenshots in `frames` are ordered earliest to latest. For current state use the latest frame; for events use the relevant frames. Reference images are supporting context, not screenshots, unless the statement requests comparison. Use the provided context and page description when present.',\n        },\n      },\n    },\n    options.modelRuntime,\n    { abortSignal: options.abortSignal },\n  );\n  const answer = systemOneResponseSchema.parse(JSON.parse(result.content))\n    .answers.result;\n  assert(answer.type === 'noul', 'Invalid Jev boolean answer');\n  return {\n    value: answer.noul >= 0.5,\n    rawResponse: result.content,\n    usage: result.usage,\n  };\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","jevBoolean","options","references","prepareReferences","frames","screenshots","Promise","frame","prepareJevImage","result","callSystemOneAI","_","index","image","answer","systemOneResponseSchema","JSON","assert"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;ACAO,MAAMI,aAAwB,OAAOC;IAC1C,MAAMC,aAAa,MAAMC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;QACzC,QAAQ;QACR,GAAGF,QAAQ,gBAAgB;IAC7B;IACA,MAAMG,SACJH,QAAQ,aAAa,EAAE,uBAAuB,QAC1C,EAAE,GACFA,QAAQ,OAAO,CAAC,kBAAkB,EAAE,SAClCA,QAAQ,OAAO,CAAC,kBAAkB,GAClC;QAACA,QAAQ,OAAO,CAAC,UAAU;KAAC;IACpC,MAAMI,cAAc,MAAMC,QAAQ,GAAG,CACnCF,OAAO,GAAG,CAAC,CAACG,QAAUC,AAAAA,IAAAA,kCAAAA,eAAAA,AAAAA,EAAgBD,MAAM,MAAM;IAEpD,MAAME,SAAS,MAAMC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EACnB;QACE,OAAO;YACL,WAAWT,QAAQ,SAAS;YAC5B,SAASA,QAAQ,aAAa,EAAE;YAChC,iBAAiBA,QAAQ,eAAe;YACxC,QAAQI,YAAY,GAAG,CAAC,CAACM,GAAGC,QAAU,CAAC,QAAQ,EAAEA,MAAM,EAAE,CAAC;YAC1D,YAAYV,WAAW,UAAU;QACnC;QACA,QAAQ;YACN,GAAGN,OAAO,WAAW,CACnBS,YAAY,GAAG,CAAC,CAACQ,OAAOD,QAAU;oBAChC,CAAC,MAAM,EAAEA,OAAO;oBAChBC,MAAM,WAAW;iBAClB,EACF;YACD,GAAGX,WAAW,MAAM;QACtB;QACA,WAAW;YACT,QAAQ;gBACN,MAAM;gBACN,cACE;YACJ;QACF;IACF,GACAD,QAAQ,YAAY,EACpB;QAAE,aAAaA,QAAQ,WAAW;IAAC;IAErC,MAAMa,SAASC,0BAAAA,uBAAAA,CAAAA,KAA6B,CAACC,KAAK,KAAK,CAACP,OAAO,OAAO,GACnE,OAAO,CAAC,MAAM;IACjBQ,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOH,AAAgB,WAAhBA,OAAO,IAAI,EAAa;IAC/B,OAAO;QACL,OAAOA,OAAO,IAAI,IAAI;QACtB,aAAaL,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/lib/ai-model/models/jev/image.js

@@ -24,59 +24,65 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
+    prepareReferences: ()=>prepareReferences,
     mapJevBox: ()=>mapJevBox,
     prepareJevImage: ()=>prepareJevImage
 });
 const external_node_buffer_namespaceObject = require("node:buffer");
 const img_namespaceObject = require("@midscene/shared/img");
 const utils_namespaceObject = require("@midscene/shared/utils");
+const index_js_namespaceObject = require("../../shared/model-locate-result/index.js");
+const multimodal_prompt_js_namespaceObject = require("../../shared/multimodal-prompt.js");
 async function prepareJevImage(url) {
     const base64 = await (0, img_namespaceObject.preProcessImageUrl)(url, true);
-    const source = await (0, img_namespaceObject.imageInfoOfBase64)(base64);
-    const scale = Math.min(1, 1568 / source.width, 1568 / source.height);
-    const contentSize = {
-        width: Math.max(1, Math.round(source.width * scale)),
-        height: Math.max(1, Math.round(source.height * scale))
-    };
-    const resized = await (0, img_namespaceObject.resizeBase64ImageToJpeg)(base64, {
-        targetSize: contentSize
-    });
-    const prepared = await (0, img_namespaceObject.padBase64ImageToMinimumSize)(resized, 280);
-    (0, utils_namespaceObject.assert)(external_node_buffer_namespaceObject.Buffer.from((0, img_namespaceObject.parseBase64)(prepared.imageBase64).body, 'base64').length <= 16777216, 'Jev image exceeds 16 MiB');
+    (0, utils_namespaceObject.assert)(external_node_buffer_namespaceObject.Buffer.from((0, img_namespaceObject.parseBase64)(base64).body, 'base64').length <= 16777216, 'Jev image exceeds 16 MiB');
     return {
-        ...prepared,
-        contentSize,
-        sourceSize: {
-            width: source.width,
-            height: source.height
-        }
+        imageBase64: base64
     };
 }
-function mapJevBox(box, image, targetSize) {
-    const left = box.left * image.width * targetSize.width / image.contentSize.width;
-    const top = box.top * image.height * targetSize.height / image.contentSize.height;
-    const right = box.right * image.width * targetSize.width / image.contentSize.width;
-    const bottom = box.bottom * image.height * targetSize.height / image.contentSize.height;
-    const center = [
-        (left + right) / 2,
-        (top + bottom) / 2
-    ];
-    (0, utils_namespaceObject.assert)(center[0] >= 0 && center[0] < targetSize.width && center[1] >= 0 && center[1] < targetSize.height, 'Jev returned a target in image padding, outside the screenshot');
+const jevBoxCodec = (0, index_js_namespaceObject.createLocateResultCodec)({
+    coordinates: {
+        shape: 'bbox',
+        normalizedBy: 1,
+        rounding: 'none',
+        boundsType: 'edges'
+    }
+});
+function mapJevBox(box, targetSize) {
+    return jevBoxCodec.toPixelResult([
+        box.left,
+        box.top,
+        box.right,
+        box.bottom
+    ], {
+        preparedSize: targetSize
+    });
+}
+async function prepareReferences(prompt) {
+    const { referenceImages } = await (0, multimodal_prompt_js_namespaceObject.prepareUserPrompt)(prompt);
+    const prepared = await Promise.all(referenceImages.map(async (image, index)=>({
+            key: `reference_${index}`,
+            name: image.name,
+            image: await prepareJevImage(image.url)
+        })));
     return {
-        center,
-        rect: {
-            left,
-            top,
-            width: right - left,
-            height: bottom - top
-        }
+        references: prepared.map(({ key, name })=>({
+                image: `\`${key}\``,
+                name
+            })),
+        images: Object.fromEntries(prepared.map(({ key, image })=>[
+                key,
+                image.imageBase64
+            ]))
     };
 }
 exports.mapJevBox = __webpack_exports__.mapJevBox;
 exports.prepareJevImage = __webpack_exports__.prepareJevImage;
+exports.prepareReferences = __webpack_exports__.prepareReferences;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
     "mapJevBox",
-    "prepareJevImage"
+    "prepareJevImage",
+    "prepareReferences"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true