@midscene/core 1.16.2 → 1.16.6-beta-20261023073257.0

package/dist/es/ai-model/connectivity/run-connectivity-test.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/connectivity/run-connectivity-test.mjs","sources":["../../../../src/ai-model/connectivity/run-connectivity-test.ts"],"sourcesContent":["import { ScreenshotItem } from '@/screenshot-item';\nimport Service from '@/service';\nimport type { UIContext } from '@/types';\nimport type { IModelConfig, TIntent } from '@midscene/shared/env';\nimport { getModelRuntime } from '../models';\nimport type { ModelRuntime } from '../models';\nimport { callAI } from '../service-caller';\nimport { systemOneResponseSchema } from '../service-caller/system-one';\nimport {\n  CONNECTIVITY_FIXTURE_IMAGE,\n  CONNECTIVITY_FIXTURE_SHOT_SIZE,\n} from './fixture';\n\nconst TEXT_EXPECTED_TOKEN = 'CONNECTIVITY_OK';\n\nexport interface ConnectivityCheckResultItem {\n  name: 'text' | 'vision' | 'aiLocate';\n  intent: TIntent;\n  modelName: string;\n  modelFamily?: string;\n  passed: boolean;\n  durationMs: number;\n  requestCount: number;\n  endpoints: string[];\n  message: string;\n}\n\nexport interface ConnectivityTestResult {\n  checks: ConnectivityCheckResultItem[];\n  requestCount: number;\n  passed: boolean;\n  message?: string;\n}\n\nexport interface ConnectivityTestConfig {\n  defaultModelConfig: IModelConfig;\n  locateModelConfig?: IModelConfig;\n  planningModelConfig: IModelConfig;\n  insightModelConfig: IModelConfig;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction hasValidCenter(value: unknown): boolean {\n  return (\n    Array.isArray(value) &&\n    value.length === 2 &&\n    isFiniteNumber(value[0]) &&\n    isFiniteNumber(value[1])\n  );\n}\n\nfunction buildFixtureContext(): UIContext {\n  return {\n    screenshot: ScreenshotItem.create(CONNECTIVITY_FIXTURE_IMAGE, Date.now()),\n    shotSize: CONNECTIVITY_FIXTURE_SHOT_SIZE,\n    shrunkShotToLogicalRatio: 1,\n  };\n}\n\nfunction buildCheckResult(\n  name: ConnectivityCheckResultItem['name'],\n  modelRuntime: ModelRuntime,\n  result: Omit<\n    ConnectivityCheckResultItem,\n    | 'name'\n    | 'intent'\n    | 'modelName'\n    | 'modelFamily'\n    | 'requestCount'\n    | 'endpoints'\n  >,\n): ConnectivityCheckResultItem {\n  const { config } = modelRuntime;\n  return {\n    name,\n    intent: config.intent,\n    modelName: config.modelName,\n    modelFamily: config.modelFamily,\n    requestCount: 0,\n    endpoints: [],\n    ...result,\n  };\n}\n\nfunction buildConnectivityModelRuntime(\n  modelConfig: IModelConfig,\n): ModelRuntime {\n  return getModelRuntime({ ...modelConfig, retryCount: 0 });\n}\n\nfunction formatConnectivityCheckName(\n  check: ConnectivityCheckResultItem,\n): string {\n  const checkName =\n    check.name === 'aiLocate'\n      ? 'AI locate check'\n      : `${check.name[0]?.toUpperCase()}${check.name.slice(1)} check`;\n  const modelLabel = check.modelName || check.intent;\n  return `${checkName} - ${modelLabel} (${check.intent})`;\n}\n\nfunction buildConnectivityFailureMessage(\n  checks: ConnectivityCheckResultItem[],\n): string {\n  const failedChecks = checks.filter((item) => !item.passed);\n  if (failedChecks.length === 0) {\n    return 'Connectivity test failed, but no failed check details were generated.';\n  }\n\n  return failedChecks\n    .map((item) => {\n      const detail = item.message || 'Failed without details.';\n      return `[${formatConnectivityCheckName(item)}]: ${detail}`;\n    })\n    .join('\\n');\n}\n\nasync function runTextConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const isJev = modelRuntime.config.modelFamily === 'jev';\n    const result = await callAI(\n      [\n        {\n          role: 'system',\n          content: 'Reply with the exact token the user asks for.',\n        },\n        {\n          role: 'user',\n          content: `Return exactly ${TEXT_EXPECTED_TOKEN}`,\n        },\n      ],\n      modelRuntime,\n      isJev\n        ? {\n            systemOneRequest: {\n              state: { token: TEXT_EXPECTED_TOKEN },\n              questions: {\n                result: {\n                  type: 'noul',\n                  instructions: `Does token equal ${TEXT_EXPECTED_TOKEN}?`,\n                },\n              },\n            },\n          }\n        : undefined,\n    );\n    const content = result.content.trim();\n    const answer = isJev\n      ? systemOneResponseSchema.parse(JSON.parse(content)).answers.result\n      : undefined;\n    const passed = isJev\n      ? answer?.type === 'noul' && answer.noul >= 0.5\n      : content.includes(TEXT_EXPECTED_TOKEN);\n    return buildCheckResult('text', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed ? '' : `Unexpected response: ${content}`,\n    });\n  } catch (error) {\n    return buildCheckResult('text', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runVisionConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    if (modelRuntime.adapter.insight.booleanFn) {\n      await new Service(buildFixtureContext()).extract(\n        { Boolean: 'Does this screenshot show a form?' },\n        modelRuntime,\n        { screenshotIncluded: true },\n        undefined,\n        undefined,\n        buildFixtureContext(),\n      );\n    } else {\n      await callAI(\n        [\n          {\n            role: 'user',\n            content: [\n              {\n                type: 'text',\n                text: 'What is the main content of this image ? It is a photo or a form ?',\n              },\n              {\n                type: 'image',\n                url: CONNECTIVITY_FIXTURE_IMAGE,\n              },\n            ],\n          },\n        ],\n        modelRuntime,\n      );\n    }\n    return buildCheckResult('vision', modelRuntime, {\n      passed: true,\n      durationMs: Date.now() - startTime,\n      message: '',\n    });\n  } catch (error) {\n    return buildCheckResult('vision', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runAiLocateConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const context = buildFixtureContext();\n    const service = new Service(context);\n    const locateResult = await service.locate(\n      { prompt: 'the main todo input box' },\n      {},\n      modelRuntime,\n    );\n    const center = locateResult.element?.center;\n    const passed = hasValidCenter(center);\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed\n        ? ''\n        : `Invalid locate result: ${JSON.stringify({\n            center,\n          })}`,\n    });\n  } catch (error) {\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nexport async function runConnectivityTest(\n  config: ConnectivityTestConfig,\n): Promise<ConnectivityTestResult> {\n  const runCheck = async (\n    modelConfig: IModelConfig,\n    check: (runtime: ModelRuntime) => Promise<ConnectivityCheckResultItem>,\n  ) => {\n    const runtime = buildConnectivityModelRuntime(modelConfig);\n    let requestCount = 0;\n    const endpoints = new Set<string>();\n    runtime.onRequest = (endpoint) => {\n      requestCount += 1;\n      if (endpoint) endpoints.add(endpoint);\n    };\n    const result = await check(runtime);\n    return { ...result, requestCount, endpoints: [...endpoints] };\n  };\n  const checks = await Promise.all([\n    runCheck(config.planningModelConfig, runTextConnectivityCheck),\n    runCheck(config.insightModelConfig, runVisionConnectivityCheck),\n    runCheck(\n      config.locateModelConfig ?? config.defaultModelConfig,\n      runAiLocateConnectivityCheck,\n    ),\n    runCheck(config.defaultModelConfig, runTextConnectivityCheck),\n  ]);\n\n  const passed = checks.every((item) => item.passed);\n  return {\n    checks,\n    requestCount: checks.reduce(\n      (total, check) => total + check.requestCount,\n      0,\n    ),\n    passed,\n    message: passed ? undefined : buildConnectivityFailureMessage(checks),\n  };\n}\n"],"names":["TEXT_EXPECTED_TOKEN","isFiniteNumber","value","Number","hasValidCenter","Array","buildFixtureContext","ScreenshotItem","CONNECTIVITY_FIXTURE_IMAGE","Date","CONNECTIVITY_FIXTURE_SHOT_SIZE","buildCheckResult","name","modelRuntime","result","config","buildConnectivityModelRuntime","modelConfig","getModelRuntime","formatConnectivityCheckName","check","checkName","modelLabel","buildConnectivityFailureMessage","checks","failedChecks","item","detail","runTextConnectivityCheck","startTime","isJev","callAI","undefined","content","answer","systemOneResponseSchema","JSON","passed","error","Error","String","runVisionConnectivityCheck","Service","runAiLocateConnectivityCheck","context","service","locateResult","center","runConnectivityTest","runCheck","runtime","requestCount","endpoints","Set","endpoint","Promise","total"],"mappings":";;;;;;AAaA,MAAMA,sBAAsB;AA4B5B,SAASC,eAAeC,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD;AACtD;AAEA,SAASE,eAAeF,KAAc;IACpC,OACEG,MAAM,OAAO,CAACH,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZD,eAAeC,KAAK,CAAC,EAAE,KACvBD,eAAeC,KAAK,CAAC,EAAE;AAE3B;AAEA,SAASI;IACP,OAAO;QACL,YAAYC,eAAe,MAAM,CAACC,4BAA4BC,KAAK,GAAG;QACtE,UAAUC;QACV,0BAA0B;IAC5B;AACF;AAEA,SAASC,iBACPC,IAAyC,EACzCC,YAA0B,EAC1BC,MAQC;IAED,MAAM,EAAEC,MAAM,EAAE,GAAGF;IACnB,OAAO;QACLD;QACA,QAAQG,OAAO,MAAM;QACrB,WAAWA,OAAO,SAAS;QAC3B,aAAaA,OAAO,WAAW;QAC/B,cAAc;QACd,WAAW,EAAE;QACb,GAAGD,MAAM;IACX;AACF;AAEA,SAASE,8BACPC,WAAyB;IAEzB,OAAOC,gBAAgB;QAAE,GAAGD,WAAW;QAAE,YAAY;IAAE;AACzD;AAEA,SAASE,4BACPC,KAAkC;IAElC,MAAMC,YACJD,AAAe,eAAfA,MAAM,IAAI,GACN,oBACA,GAAGA,MAAM,IAAI,CAAC,EAAE,EAAE,gBAAgBA,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IACnE,MAAME,aAAaF,MAAM,SAAS,IAAIA,MAAM,MAAM;IAClD,OAAO,GAAGC,UAAU,GAAG,EAAEC,WAAW,EAAE,EAAEF,MAAM,MAAM,CAAC,CAAC,CAAC;AACzD;AAEA,SAASG,gCACPC,MAAqC;IAErC,MAAMC,eAAeD,OAAO,MAAM,CAAC,CAACE,OAAS,CAACA,KAAK,MAAM;IACzD,IAAID,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAO;IAGT,OAAOA,aACJ,GAAG,CAAC,CAACC;QACJ,MAAMC,SAASD,KAAK,OAAO,IAAI;QAC/B,OAAO,CAAC,CAAC,EAAEP,4BAA4BO,MAAM,GAAG,EAAEC,QAAQ;IAC5D,GACC,IAAI,CAAC;AACV;AAEA,eAAeC,yBACbf,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMqB,QAAQjB,AAAoC,UAApCA,aAAa,MAAM,CAAC,WAAW;QAC7C,MAAMC,SAAS,MAAMiB,OACnB;YACE;gBACE,MAAM;gBACN,SAAS;YACX;YACA;gBACE,MAAM;gBACN,SAAS,CAAC,eAAe,EAAE/B,qBAAqB;YAClD;SACD,EACDa,cACAiB,QACI;YACE,kBAAkB;gBAChB,OAAO;oBAAE,OAAO9B;gBAAoB;gBACpC,WAAW;oBACT,QAAQ;wBACN,MAAM;wBACN,cAAc,CAAC,iBAAiB,EAAEA,oBAAoB,CAAC,CAAC;oBAC1D;gBACF;YACF;QACF,IACAgC;QAEN,MAAMC,UAAUnB,OAAO,OAAO,CAAC,IAAI;QACnC,MAAMoB,SAASJ,QACXK,wBAAwB,KAAK,CAACC,KAAK,KAAK,CAACH,UAAU,OAAO,CAAC,MAAM,GACjED;QACJ,MAAMK,SAASP,QACXI,QAAQ,SAAS,UAAUA,OAAO,IAAI,IAAI,MAC1CD,QAAQ,QAAQ,CAACjC;QACrB,OAAOW,iBAAiB,QAAQE,cAAc;YAC5CwB;YACA,YAAY5B,KAAK,GAAG,KAAKoB;YACzB,SAASQ,SAAS,KAAK,CAAC,qBAAqB,EAAEJ,SAAS;QAC1D;IACF,EAAE,OAAOK,OAAO;QACd,OAAO3B,iBAAiB,QAAQE,cAAc;YAC5C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASS,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeG,2BACb5B,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,IAAII,aAAa,OAAO,CAAC,OAAO,CAAC,SAAS,EACxC,MAAM,IAAI6B,UAAQpC,uBAAuB,OAAO,CAC9C;YAAE,SAAS;QAAoC,GAC/CO,cACA;YAAE,oBAAoB;QAAK,GAC3BmB,QACAA,QACA1B;aAGF,MAAMyB,OACJ;YACE;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM;oBACR;oBACA;wBACE,MAAM;wBACN,KAAKvB;oBACP;iBACD;YACH;SACD,EACDK;QAGJ,OAAOF,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAAS;QACX;IACF,EAAE,OAAOS,OAAO;QACd,OAAO3B,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASS,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeK,6BACb9B,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMmC,UAAUtC;QAChB,MAAMuC,UAAU,IAAIH,UAAQE;QAC5B,MAAME,eAAe,MAAMD,QAAQ,MAAM,CACvC;YAAE,QAAQ;QAA0B,GACpC,CAAC,GACDhC;QAEF,MAAMkC,SAASD,aAAa,OAAO,EAAE;QACrC,MAAMT,SAASjC,eAAe2C;QAC9B,OAAOpC,iBAAiB,YAAYE,cAAc;YAChDwB;YACA,YAAY5B,KAAK,GAAG,KAAKoB;YACzB,SAASQ,SACL,KACA,CAAC,uBAAuB,EAAED,KAAK,SAAS,CAAC;gBACvCW;YACF,IAAI;QACV;IACF,EAAE,OAAOT,OAAO;QACd,OAAO3B,iBAAiB,YAAYE,cAAc;YAChD,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASS,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEO,eAAeU,oBACpBjC,MAA8B;IAE9B,MAAMkC,WAAW,OACfhC,aACAG;QAEA,MAAM8B,UAAUlC,8BAA8BC;QAC9C,IAAIkC,eAAe;QACnB,MAAMC,YAAY,IAAIC;QACtBH,QAAQ,SAAS,GAAG,CAACI;YACnBH,gBAAgB;YAChB,IAAIG,UAAUF,UAAU,GAAG,CAACE;QAC9B;QACA,MAAMxC,SAAS,MAAMM,MAAM8B;QAC3B,OAAO;YAAE,GAAGpC,MAAM;YAAEqC;YAAc,WAAW;mBAAIC;aAAU;QAAC;IAC9D;IACA,MAAM5B,SAAS,MAAM+B,QAAQ,GAAG,CAAC;QAC/BN,SAASlC,OAAO,mBAAmB,EAAEa;QACrCqB,SAASlC,OAAO,kBAAkB,EAAE0B;QACpCQ,SACElC,OAAO,iBAAiB,IAAIA,OAAO,kBAAkB,EACrD4B;QAEFM,SAASlC,OAAO,kBAAkB,EAAEa;KACrC;IAED,MAAMS,SAASb,OAAO,KAAK,CAAC,CAACE,OAASA,KAAK,MAAM;IACjD,OAAO;QACLF;QACA,cAAcA,OAAO,MAAM,CACzB,CAACgC,OAAOpC,QAAUoC,QAAQpC,MAAM,YAAY,EAC5C;QAEFiB;QACA,SAASA,SAASL,SAAYT,gCAAgCC;IAChE;AACF"}
+{"version":3,"file":"ai-model/connectivity/run-connectivity-test.mjs","sources":["../../../../src/ai-model/connectivity/run-connectivity-test.ts"],"sourcesContent":["import { ScreenshotItem } from '@/screenshot-item';\nimport Service from '@/service';\nimport type { UIContext } from '@/types';\nimport type { IModelConfig, TIntent } from '@midscene/shared/env';\nimport { getModelRuntime } from '../models';\nimport type { ModelRuntime } from '../models';\nimport { callGenerativeAI, callSystemOneAI } from '../service-caller';\nimport { systemOneResponseSchema } from '../service-caller/system-one/schema';\nimport {\n  CONNECTIVITY_FIXTURE_IMAGE,\n  CONNECTIVITY_FIXTURE_SHOT_SIZE,\n} from './fixture';\n\nconst TEXT_EXPECTED_TOKEN = 'CONNECTIVITY_OK';\n\nexport interface ConnectivityCheckResultItem {\n  name: 'text' | 'vision' | 'aiLocate';\n  intent: TIntent;\n  modelName: string;\n  modelFamily?: string;\n  passed: boolean;\n  durationMs: number;\n  requestCount: number;\n  endpoints: string[];\n  message: string;\n}\n\nexport interface ConnectivityTestResult {\n  checks: ConnectivityCheckResultItem[];\n  requestCount: number;\n  passed: boolean;\n  message?: string;\n}\n\nexport interface ConnectivityTestConfig {\n  defaultModelConfig: IModelConfig;\n  locateModelConfig?: IModelConfig;\n  planningModelConfig: IModelConfig;\n  insightModelConfig: IModelConfig;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction hasValidCenter(value: unknown): boolean {\n  return (\n    Array.isArray(value) &&\n    value.length === 2 &&\n    isFiniteNumber(value[0]) &&\n    isFiniteNumber(value[1])\n  );\n}\n\nfunction buildFixtureContext(): UIContext {\n  return {\n    screenshot: ScreenshotItem.create(CONNECTIVITY_FIXTURE_IMAGE, Date.now()),\n    shotSize: CONNECTIVITY_FIXTURE_SHOT_SIZE,\n    shrunkShotToLogicalRatio: 1,\n  };\n}\n\nfunction buildCheckResult(\n  name: ConnectivityCheckResultItem['name'],\n  modelRuntime: ModelRuntime,\n  result: Omit<\n    ConnectivityCheckResultItem,\n    | 'name'\n    | 'intent'\n    | 'modelName'\n    | 'modelFamily'\n    | 'requestCount'\n    | 'endpoints'\n  >,\n): ConnectivityCheckResultItem {\n  const { config } = modelRuntime;\n  return {\n    name,\n    intent: config.intent,\n    modelName: config.modelName,\n    modelFamily: config.modelFamily,\n    requestCount: 0,\n    endpoints: [],\n    ...result,\n  };\n}\n\nfunction buildConnectivityModelRuntime(\n  modelConfig: IModelConfig,\n): ModelRuntime {\n  return getModelRuntime({ ...modelConfig, retryCount: 0 });\n}\n\nfunction formatConnectivityCheckName(\n  check: ConnectivityCheckResultItem,\n): string {\n  const checkName =\n    check.name === 'aiLocate'\n      ? 'AI locate check'\n      : `${check.name[0]?.toUpperCase()}${check.name.slice(1)} check`;\n  const modelLabel = check.modelName || check.intent;\n  return `${checkName} - ${modelLabel} (${check.intent})`;\n}\n\nfunction buildConnectivityFailureMessage(\n  checks: ConnectivityCheckResultItem[],\n): string {\n  const failedChecks = checks.filter((item) => !item.passed);\n  if (failedChecks.length === 0) {\n    return 'Connectivity test failed, but no failed check details were generated.';\n  }\n\n  return failedChecks\n    .map((item) => {\n      const detail = item.message || 'Failed without details.';\n      return `[${formatConnectivityCheckName(item)}]: ${detail}`;\n    })\n    .join('\\n');\n}\n\nasync function runTextConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const isJev = modelRuntime.config.modelFamily === 'jev';\n    const result = await (isJev\n      ? callSystemOneAI(\n          {\n            state: { token: TEXT_EXPECTED_TOKEN },\n            questions: {\n              result: {\n                type: 'noul',\n                instructions: `Does token equal ${TEXT_EXPECTED_TOKEN}?`,\n              },\n            },\n          },\n          modelRuntime,\n        )\n      : callGenerativeAI(\n          [\n            {\n              role: 'system',\n              content: 'Reply with the exact token the user asks for.',\n            },\n            {\n              role: 'user',\n              content: `Return exactly ${TEXT_EXPECTED_TOKEN}`,\n            },\n          ],\n          modelRuntime,\n        ));\n    const content = result.content.trim();\n    const answer = isJev\n      ? systemOneResponseSchema.parse(JSON.parse(content)).answers.result\n      : undefined;\n    const passed = isJev\n      ? answer?.type === 'noul' && answer.noul >= 0.5\n      : content.includes(TEXT_EXPECTED_TOKEN);\n    return buildCheckResult('text', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed ? '' : `Unexpected response: ${content}`,\n    });\n  } catch (error) {\n    return buildCheckResult('text', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runVisionConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    if (modelRuntime.config.modelFamily === 'jev') {\n      await callSystemOneAI(\n        {\n          state: {},\n          images: { screenshot: CONNECTIVITY_FIXTURE_IMAGE },\n          questions: {\n            result: {\n              type: 'noul',\n              instructions: 'Does `screenshot` show a form?',\n            },\n          },\n        },\n        modelRuntime,\n      );\n    } else {\n      await callGenerativeAI(\n        [\n          {\n            role: 'user',\n            content: [\n              {\n                type: 'text',\n                text: 'What is the main content of this image ? It is a photo or a form ?',\n              },\n              {\n                type: 'image',\n                url: CONNECTIVITY_FIXTURE_IMAGE,\n              },\n            ],\n          },\n        ],\n        modelRuntime,\n      );\n    }\n    return buildCheckResult('vision', modelRuntime, {\n      passed: true,\n      durationMs: Date.now() - startTime,\n      message: '',\n    });\n  } catch (error) {\n    return buildCheckResult('vision', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runAiLocateConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const context = buildFixtureContext();\n    const service = new Service(context);\n    const locateResult = await service.locate(\n      { prompt: 'the main todo input box' },\n      {},\n      modelRuntime,\n    );\n    const center = locateResult.element?.center;\n    const passed = hasValidCenter(center);\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed\n        ? ''\n        : `Invalid locate result: ${JSON.stringify({\n            center,\n          })}`,\n    });\n  } catch (error) {\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nexport async function runConnectivityTest(\n  config: ConnectivityTestConfig,\n): Promise<ConnectivityTestResult> {\n  const runCheck = async (\n    modelConfig: IModelConfig,\n    check: (runtime: ModelRuntime) => Promise<ConnectivityCheckResultItem>,\n  ) => {\n    const runtime = buildConnectivityModelRuntime(modelConfig);\n    let requestCount = 0;\n    const endpoints = new Set<string>();\n    runtime.onRequest = (endpoint) => {\n      requestCount += 1;\n      if (endpoint) endpoints.add(endpoint);\n    };\n    const result = await check(runtime);\n    return { ...result, requestCount, endpoints: [...endpoints] };\n  };\n  const checks = await Promise.all([\n    runCheck(config.planningModelConfig, runTextConnectivityCheck),\n    runCheck(config.insightModelConfig, runVisionConnectivityCheck),\n    runCheck(\n      config.locateModelConfig ?? config.defaultModelConfig,\n      runAiLocateConnectivityCheck,\n    ),\n    runCheck(config.defaultModelConfig, runTextConnectivityCheck),\n  ]);\n\n  const passed = checks.every((item) => item.passed);\n  return {\n    checks,\n    requestCount: checks.reduce(\n      (total, check) => total + check.requestCount,\n      0,\n    ),\n    passed,\n    message: passed ? undefined : buildConnectivityFailureMessage(checks),\n  };\n}\n"],"names":["TEXT_EXPECTED_TOKEN","isFiniteNumber","value","Number","hasValidCenter","Array","buildFixtureContext","ScreenshotItem","CONNECTIVITY_FIXTURE_IMAGE","Date","CONNECTIVITY_FIXTURE_SHOT_SIZE","buildCheckResult","name","modelRuntime","result","config","buildConnectivityModelRuntime","modelConfig","getModelRuntime","formatConnectivityCheckName","check","checkName","modelLabel","buildConnectivityFailureMessage","checks","failedChecks","item","detail","runTextConnectivityCheck","startTime","isJev","callSystemOneAI","callGenerativeAI","content","answer","systemOneResponseSchema","JSON","undefined","passed","error","Error","String","runVisionConnectivityCheck","runAiLocateConnectivityCheck","context","service","Service","locateResult","center","runConnectivityTest","runCheck","runtime","requestCount","endpoints","Set","endpoint","Promise","total"],"mappings":";;;;;;AAaA,MAAMA,sBAAsB;AA4B5B,SAASC,eAAeC,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD;AACtD;AAEA,SAASE,eAAeF,KAAc;IACpC,OACEG,MAAM,OAAO,CAACH,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZD,eAAeC,KAAK,CAAC,EAAE,KACvBD,eAAeC,KAAK,CAAC,EAAE;AAE3B;AAEA,SAASI;IACP,OAAO;QACL,YAAYC,eAAe,MAAM,CAACC,4BAA4BC,KAAK,GAAG;QACtE,UAAUC;QACV,0BAA0B;IAC5B;AACF;AAEA,SAASC,iBACPC,IAAyC,EACzCC,YAA0B,EAC1BC,MAQC;IAED,MAAM,EAAEC,MAAM,EAAE,GAAGF;IACnB,OAAO;QACLD;QACA,QAAQG,OAAO,MAAM;QACrB,WAAWA,OAAO,SAAS;QAC3B,aAAaA,OAAO,WAAW;QAC/B,cAAc;QACd,WAAW,EAAE;QACb,GAAGD,MAAM;IACX;AACF;AAEA,SAASE,8BACPC,WAAyB;IAEzB,OAAOC,gBAAgB;QAAE,GAAGD,WAAW;QAAE,YAAY;IAAE;AACzD;AAEA,SAASE,4BACPC,KAAkC;IAElC,MAAMC,YACJD,AAAe,eAAfA,MAAM,IAAI,GACN,oBACA,GAAGA,MAAM,IAAI,CAAC,EAAE,EAAE,gBAAgBA,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IACnE,MAAME,aAAaF,MAAM,SAAS,IAAIA,MAAM,MAAM;IAClD,OAAO,GAAGC,UAAU,GAAG,EAAEC,WAAW,EAAE,EAAEF,MAAM,MAAM,CAAC,CAAC,CAAC;AACzD;AAEA,SAASG,gCACPC,MAAqC;IAErC,MAAMC,eAAeD,OAAO,MAAM,CAAC,CAACE,OAAS,CAACA,KAAK,MAAM;IACzD,IAAID,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAO;IAGT,OAAOA,aACJ,GAAG,CAAC,CAACC;QACJ,MAAMC,SAASD,KAAK,OAAO,IAAI;QAC/B,OAAO,CAAC,CAAC,EAAEP,4BAA4BO,MAAM,GAAG,EAAEC,QAAQ;IAC5D,GACC,IAAI,CAAC;AACV;AAEA,eAAeC,yBACbf,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMqB,QAAQjB,AAAoC,UAApCA,aAAa,MAAM,CAAC,WAAW;QAC7C,MAAMC,SAAS,MAAOgB,CAAAA,QAClBC,gBACE;YACE,OAAO;gBAAE,OAAO/B;YAAoB;YACpC,WAAW;gBACT,QAAQ;oBACN,MAAM;oBACN,cAAc,CAAC,iBAAiB,EAAEA,oBAAoB,CAAC,CAAC;gBAC1D;YACF;QACF,GACAa,gBAEFmB,iBACE;YACE;gBACE,MAAM;gBACN,SAAS;YACX;YACA;gBACE,MAAM;gBACN,SAAS,CAAC,eAAe,EAAEhC,qBAAqB;YAClD;SACD,EACDa,aAAY;QAElB,MAAMoB,UAAUnB,OAAO,OAAO,CAAC,IAAI;QACnC,MAAMoB,SAASJ,QACXK,wBAAwB,KAAK,CAACC,KAAK,KAAK,CAACH,UAAU,OAAO,CAAC,MAAM,GACjEI;QACJ,MAAMC,SAASR,QACXI,QAAQ,SAAS,UAAUA,OAAO,IAAI,IAAI,MAC1CD,QAAQ,QAAQ,CAACjC;QACrB,OAAOW,iBAAiB,QAAQE,cAAc;YAC5CyB;YACA,YAAY7B,KAAK,GAAG,KAAKoB;YACzB,SAASS,SAAS,KAAK,CAAC,qBAAqB,EAAEL,SAAS;QAC1D;IACF,EAAE,OAAOM,OAAO;QACd,OAAO5B,iBAAiB,QAAQE,cAAc;YAC5C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASU,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeG,2BACb7B,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,IAAII,AAAoC,UAApCA,aAAa,MAAM,CAAC,WAAW,EACjC,MAAMkB,gBACJ;YACE,OAAO,CAAC;YACR,QAAQ;gBAAE,YAAYvB;YAA2B;YACjD,WAAW;gBACT,QAAQ;oBACN,MAAM;oBACN,cAAc;gBAChB;YACF;QACF,GACAK;aAGF,MAAMmB,iBACJ;YACE;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM;oBACR;oBACA;wBACE,MAAM;wBACN,KAAKxB;oBACP;iBACD;YACH;SACD,EACDK;QAGJ,OAAOF,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAAS;QACX;IACF,EAAE,OAAOU,OAAO;QACd,OAAO5B,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASU,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeI,6BACb9B,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMmC,UAAUtC;QAChB,MAAMuC,UAAU,IAAIC,UAAQF;QAC5B,MAAMG,eAAe,MAAMF,QAAQ,MAAM,CACvC;YAAE,QAAQ;QAA0B,GACpC,CAAC,GACDhC;QAEF,MAAMmC,SAASD,aAAa,OAAO,EAAE;QACrC,MAAMT,SAASlC,eAAe4C;QAC9B,OAAOrC,iBAAiB,YAAYE,cAAc;YAChDyB;YACA,YAAY7B,KAAK,GAAG,KAAKoB;YACzB,SAASS,SACL,KACA,CAAC,uBAAuB,EAAEF,KAAK,SAAS,CAAC;gBACvCY;YACF,IAAI;QACV;IACF,EAAE,OAAOT,OAAO;QACd,OAAO5B,iBAAiB,YAAYE,cAAc;YAChD,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASU,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEO,eAAeU,oBACpBlC,MAA8B;IAE9B,MAAMmC,WAAW,OACfjC,aACAG;QAEA,MAAM+B,UAAUnC,8BAA8BC;QAC9C,IAAImC,eAAe;QACnB,MAAMC,YAAY,IAAIC;QACtBH,QAAQ,SAAS,GAAG,CAACI;YACnBH,gBAAgB;YAChB,IAAIG,UAAUF,UAAU,GAAG,CAACE;QAC9B;QACA,MAAMzC,SAAS,MAAMM,MAAM+B;QAC3B,OAAO;YAAE,GAAGrC,MAAM;YAAEsC;YAAc,WAAW;mBAAIC;aAAU;QAAC;IAC9D;IACA,MAAM7B,SAAS,MAAMgC,QAAQ,GAAG,CAAC;QAC/BN,SAASnC,OAAO,mBAAmB,EAAEa;QACrCsB,SAASnC,OAAO,kBAAkB,EAAE2B;QACpCQ,SACEnC,OAAO,iBAAiB,IAAIA,OAAO,kBAAkB,EACrD4B;QAEFO,SAASnC,OAAO,kBAAkB,EAAEa;KACrC;IAED,MAAMU,SAASd,OAAO,KAAK,CAAC,CAACE,OAASA,KAAK,MAAM;IACjD,OAAO;QACLF;QACA,cAAcA,OAAO,MAAM,CACzB,CAACiC,OAAOrC,QAAUqC,QAAQrC,MAAM,YAAY,EAC5C;QAEFkB;QACA,SAASA,SAASD,SAAYd,gCAAgCC;IAChE;AACF"}

package/dist/es/ai-model/index.mjs

@@ -1,9 +1,11 @@
-import { AIResponseParseError, callAI, callAIWithObjectResponse, callAIWithStringResponse } from "./service-caller/index.mjs";
+import { AIResponseParseError, callAIWithObjectResponse, callAIWithStringResponse, callGenerativeAI, callSystemOneAI } from "./service-caller/index.mjs";
 import { getModelRuntime } from "./models/index.mjs";
+import { SIWC_AUTH_STORAGE_KEY, createSiwcAuthorization, exchangeSiwcCode, parseSiwcCallback, refreshSiwcCredentials } from "./service-caller/openai/siwc/index.mjs";
 import { runConnectivityTest } from "./connectivity/index.mjs";
 import { convertRecordLogIntoMarkdown, createRecorderMarkdownReplayPrompt, generatePlaywrightTest, generatePlaywrightTestStream, generateRecorderMarkdownReplay, generateRecorderSessionMetadata, generateRecorderYamlTest, generateRecorderYamlTestStream, generateYamlTest, generateYamlTestStream } from "./workflows/recorder-generation/index.mjs";
 import { AiLocateElement, AiLocateSection } from "./workflows/grounding/index.mjs";
 import { AiExtractElementInfo, AiJudgeOrderSensitive } from "./workflows/insight/index.mjs";
 import { ConversationHistory, standardPlan } from "./workflows/planning/index.mjs";
 import { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, dumpActionParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, parseActionParam } from "../common.mjs";
-export { AIResponseParseError, AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, callAI, callAIWithObjectResponse, callAIWithStringResponse, convertRecordLogIntoMarkdown, createRecorderMarkdownReplayPrompt, dumpActionParam, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateRecorderMarkdownReplay, generateRecorderSessionMetadata, generateRecorderYamlTest, generateRecorderYamlTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, getModelRuntime, parseActionParam, runConnectivityTest, standardPlan };
+import { SiwcError, tokenResponseError, withSiwcStage } from "./service-caller/openai/siwc/errors.mjs";
+export { AIResponseParseError, AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SIWC_AUTH_STORAGE_KEY, SiwcError, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, callAIWithObjectResponse, callAIWithStringResponse, callGenerativeAI, callSystemOneAI, convertRecordLogIntoMarkdown, createRecorderMarkdownReplayPrompt, createSiwcAuthorization, dumpActionParam, exchangeSiwcCode, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateRecorderMarkdownReplay, generateRecorderSessionMetadata, generateRecorderYamlTest, generateRecorderYamlTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, getModelRuntime, parseActionParam, parseSiwcCallback, refreshSiwcCredentials, runConnectivityTest, standardPlan, tokenResponseError, withSiwcStage };

package/dist/es/ai-model/model-adapter/resolve.mjs

@@ -28,7 +28,8 @@ function resolveImagePreprocess(imagePreprocess) {
 }
 class ResolvedModelAdapter {
     constructor(config, modelFamily){
-        _define_property(this, "supportedTasks", void 0);
+        _define_property(this, "modelType", void 0);
+        _define_property(this, "supportedTaskTypes", void 0);
         _define_property(this, "supportedProtocols", void 0);
         _define_property(this, "jsonParser", void 0);
         _define_property(this, "chatCompletion", void 0);
@@ -39,7 +40,8 @@ class ResolvedModelAdapter {
         _define_property(this, "insight", void 0);
         _define_property(this, "planning", void 0);
         _define_property(this, "locate", void 0);
-        this.supportedTasks = config.supportedTasks;
+        this.modelType = config.modelType ?? 'generative';
+        this.supportedTaskTypes = config.supportedTaskTypes;
         this.supportedProtocols = config.supportedProtocols ?? [
             'openai-chat'
         ];

package/dist/es/ai-model/model-adapter/resolve.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/resolve.mjs","sources":["../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["import { parseModelResponseJson } from '../shared/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { buildDefaultCodexAppServerParams } from './codex-app-server';\nimport { resolveInsight } from './insight';\nimport type { InsightAdapter } from './insight-protocol';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport { resolveResponses } from './responses';\nimport type {\n  BuildCodexAppServerParams,\n  ChatCompletionAdapter,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n  ResolveImageDetail,\n  ResponsesAdapter,\n} from './types';\n\nconst defaultImageDetail: ResolveImageDetail = ({ imageDetail }) =>\n  imageDetail ?? 'high';\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return parseModelResponseJson;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly supportedTasks: ModelAdapter['supportedTasks'];\n  readonly supportedProtocols: ModelAdapter['supportedProtocols'];\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly resolveImageDetail: ResolveImageDetail;\n  readonly responses: ResponsesAdapter;\n  readonly buildCodexAppServerParams: BuildCodexAppServerParams;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly insight: InsightAdapter;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.supportedTasks = config.supportedTasks;\n    this.supportedProtocols = config.supportedProtocols ?? ['openai-chat'];\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.resolveImageDetail = config.resolveImageDetail ?? defaultImageDetail;\n    this.responses = resolveResponses(config.responses);\n    this.buildCodexAppServerParams =\n      config.buildCodexAppServerParams ?? buildDefaultCodexAppServerParams;\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    this.insight = resolveInsight(config.insight, {\n      jsonParser: this.jsonParser,\n    });\n    const customPlanner =\n      config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n    const resolvedCustomPlanner = customPlanner\n      ? resolveCustomPlanningDefinition(customPlanner)\n      : undefined;\n    this.locate = resolveLocate(config.locate, resolvedCustomPlanner, {\n      jsonParser: this.jsonParser,\n    });\n    this.planning = resolvePlanning(\n      config.planning,\n      resolvedCustomPlanner,\n      { jsonParser: this.jsonParser },\n      this.locate.kind === 'standard'\n        ? this.locate.element.resultCodec\n        : undefined,\n    );\n  }\n}\n"],"names":["defaultImageDetail","imageDetail","resolveJsonParser","jsonParser","parseModelResponseJson","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","resolveResponses","buildDefaultCodexAppServerParams","resolveInsight","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolveLocate","resolvePlanning"],"mappings":";;;;;;;;;;;;;;;;;AAqBA,MAAMA,qBAAyC,CAAC,EAAEC,WAAW,EAAE,GAC7DA,eAAe;AAEjB,SAASC,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAaX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QAZjE,uBAAS,kBAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,aAAT;QACA,uBAAS,6BAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,WAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,cAAc,GAAGD,OAAO,cAAc;QAC3C,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAI;YAAC;SAAc;QACtE,IAAI,CAAC,UAAU,GAAGP,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,sBAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAIT;QACvD,IAAI,CAAC,SAAS,GAAGY,iBAAiBH,OAAO,SAAS;QAClD,IAAI,CAAC,yBAAyB,GAC5BA,OAAO,yBAAyB,IAAII;QACtC,IAAI,CAAC,eAAe,GAAGP,uBAAuBG,OAAO,eAAe;QACpE,IAAI,CAAC,OAAO,GAAGK,eAAeL,OAAO,OAAO,EAAE;YAC5C,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,MAAMM,gBACJN,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGO;QACjE,MAAMC,wBAAwBF,gBAC1BG,gCAAgCH,iBAChCC;QACJ,IAAI,CAAC,MAAM,GAAGG,cAAcV,OAAO,MAAM,EAAEQ,uBAAuB;YAChE,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,IAAI,CAAC,QAAQ,GAAGG,gBACdX,OAAO,QAAQ,EACfQ,uBACA;YAAE,YAAY,IAAI,CAAC,UAAU;QAAC,GAC9B,AAAqB,eAArB,IAAI,CAAC,MAAM,CAAC,IAAI,GACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,GAC/BD;IAER;AACF"}
+{"version":3,"file":"ai-model/model-adapter/resolve.mjs","sources":["../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["import { parseModelResponseJson } from '../shared/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { buildDefaultCodexAppServerParams } from './codex-app-server';\nimport { resolveInsight } from './insight';\nimport type { InsightAdapter } from './insight-protocol';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport { resolveResponses } from './responses';\nimport type {\n  BuildCodexAppServerParams,\n  ChatCompletionAdapter,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n  ResolveImageDetail,\n  ResponsesAdapter,\n} from './types';\n\nconst defaultImageDetail: ResolveImageDetail = ({ imageDetail }) =>\n  imageDetail ?? 'high';\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return parseModelResponseJson;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly modelType: ModelAdapter['modelType'];\n  readonly supportedTaskTypes: ModelAdapter['supportedTaskTypes'];\n  readonly supportedProtocols: ModelAdapter['supportedProtocols'];\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly resolveImageDetail: ResolveImageDetail;\n  readonly responses: ResponsesAdapter;\n  readonly buildCodexAppServerParams: BuildCodexAppServerParams;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly insight: InsightAdapter;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.modelType = config.modelType ?? 'generative';\n    this.supportedTaskTypes = config.supportedTaskTypes;\n    this.supportedProtocols = config.supportedProtocols ?? ['openai-chat'];\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.resolveImageDetail = config.resolveImageDetail ?? defaultImageDetail;\n    this.responses = resolveResponses(config.responses);\n    this.buildCodexAppServerParams =\n      config.buildCodexAppServerParams ?? buildDefaultCodexAppServerParams;\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    this.insight = resolveInsight(config.insight, {\n      jsonParser: this.jsonParser,\n    });\n    const customPlanner =\n      config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n    const resolvedCustomPlanner = customPlanner\n      ? resolveCustomPlanningDefinition(customPlanner)\n      : undefined;\n    this.locate = resolveLocate(config.locate, resolvedCustomPlanner, {\n      jsonParser: this.jsonParser,\n    });\n    this.planning = resolvePlanning(\n      config.planning,\n      resolvedCustomPlanner,\n      { jsonParser: this.jsonParser },\n      this.locate.kind === 'standard'\n        ? this.locate.element.resultCodec\n        : undefined,\n    );\n  }\n}\n"],"names":["defaultImageDetail","imageDetail","resolveJsonParser","jsonParser","parseModelResponseJson","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","resolveResponses","buildDefaultCodexAppServerParams","resolveInsight","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolveLocate","resolvePlanning"],"mappings":";;;;;;;;;;;;;;;;;AAqBA,MAAMA,qBAAyC,CAAC,EAAEC,WAAW,EAAE,GAC7DA,eAAe;AAEjB,SAASC,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAcX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QAbjE,uBAAS,aAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,sBAAT;QACA,uBAAS,aAAT;QACA,uBAAS,6BAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,WAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,SAAS,GAAGD,OAAO,SAAS,IAAI;QACrC,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB;QACnD,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAI;YAAC;SAAc;QACtE,IAAI,CAAC,UAAU,GAAGP,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,sBAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,kBAAkB,GAAGA,OAAO,kBAAkB,IAAIT;QACvD,IAAI,CAAC,SAAS,GAAGY,iBAAiBH,OAAO,SAAS;QAClD,IAAI,CAAC,yBAAyB,GAC5BA,OAAO,yBAAyB,IAAII;QACtC,IAAI,CAAC,eAAe,GAAGP,uBAAuBG,OAAO,eAAe;QACpE,IAAI,CAAC,OAAO,GAAGK,eAAeL,OAAO,OAAO,EAAE;YAC5C,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,MAAMM,gBACJN,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGO;QACjE,MAAMC,wBAAwBF,gBAC1BG,gCAAgCH,iBAChCC;QACJ,IAAI,CAAC,MAAM,GAAGG,cAAcV,OAAO,MAAM,EAAEQ,uBAAuB;YAChE,YAAY,IAAI,CAAC,UAAU;QAC7B;QACA,IAAI,CAAC,QAAQ,GAAGG,gBACdX,OAAO,QAAQ,EACfQ,uBACA;YAAE,YAAY,IAAI,CAAC,UAAU;QAAC,GAC9B,AAAqB,eAArB,IAAI,CAAC,MAAM,CAAC,IAAI,GACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,GAC/BD;IAER;AACF"}

package/dist/es/ai-model/models/jev/adapter.mjs

@@ -1,114 +1,10 @@
-import { userPromptToString } from "../../../common.mjs";
-import { assert } from "@midscene/shared/utils";
-import { callAI } from "../../service-caller/index.mjs";
-import { systemOneResponseSchema } from "../../service-caller/system-one.mjs";
-import { prepareUserPrompt } from "../../shared/multimodal-prompt.mjs";
-import { mapJevBox, prepareJevImage } from "./image.mjs";
+import { jevBoolean } from "./boolean.mjs";
+import { jevLocate } from "./locate.mjs";
 import { jevPlan } from "./planning.mjs";
-async function prepareReferences(prompt) {
-    const { referenceImages } = await prepareUserPrompt(prompt);
-    const prepared = await Promise.all(referenceImages.map(async (image, index)=>({
-            key: `reference_${index}`,
-            name: image.name,
-            image: await prepareJevImage(image.url)
-        })));
-    return {
-        references: prepared.map(({ key, name })=>({
-                image: `\`${key}\``,
-                name
-            })),
-        images: Object.fromEntries(prepared.map(({ key, image })=>[
-                key,
-                image.imageBase64
-            ]))
-    };
-}
-const locate = async ({ targetElementDescription, locateImage, options })=>{
-    const image = await prepareJevImage(locateImage.imageBase64);
-    const references = await prepareReferences(targetElementDescription);
-    const target = userPromptToString(targetElementDescription);
-    const response = await callAI([], options.modelRuntime, {
-        abortSignal: options.abortSignal,
-        systemOneRequest: {
-            state: {
-                target,
-                references: references.references
-            },
-            images: {
-                screenshot: image.imageBase64,
-                ...references.images
-            },
-            questions: {
-                present: {
-                    type: 'noul',
-                    instructions: 'Is the target described in `target` present in `screenshot`? Reference images, if any, describe the target; they are not the screenshot.'
-                },
-                target: {
-                    type: 'box',
-                    instructions: 'The single target described in `target`, in `screenshot`.'
-                }
-            }
-        }
-    });
-    const { answers } = systemOneResponseSchema.parse(JSON.parse(response.content));
-    assert('noul' === answers.present.type && 'box' === answers.target.type, 'Invalid Jev locate answers');
-    return {
-        rawResponse: response.content,
-        usage: response.usage,
-        ...answers.present.noul >= 0.5 ? {
-            locatedPixelResult: mapJevBox(answers.target.box, image, locateImage)
-        } : {
-            errors: [
-                'The target is not present in the screenshot'
-            ]
-        }
-    };
-};
-const adapter_boolean = async (options)=>{
-    const references = await prepareReferences({
-        prompt: '',
-        ...options.multimodalPrompt
-    });
-    const frames = options.extractOption?.screenshotIncluded === false ? [] : options.context.screenshotSequence?.length ? options.context.screenshotSequence : [
-        options.context.screenshot
-    ];
-    const screenshots = await Promise.all(frames.map((frame)=>prepareJevImage(frame.base64)));
-    const result = await callAI([], options.modelRuntime, {
-        abortSignal: options.abortSignal,
-        systemOneRequest: {
-            state: {
-                statement: options.dataQuery,
-                context: options.extractOption?.context,
-                pageDescription: options.pageDescription,
-                frames: screenshots.map((_, index)=>`\`frame_${index}\``),
-                references: references.references
-            },
-            images: {
-                ...Object.fromEntries(screenshots.map((image, index)=>[
-                        `frame_${index}`,
-                        image.imageBase64
-                    ])),
-                ...references.images
-            },
-            questions: {
-                result: {
-                    type: 'noul',
-                    instructions: "Is `statement` true? Screenshots in `frames` are ordered earliest to latest. For current state use the latest frame; for events use the relevant frames. Reference images are supporting context, not screenshots, unless the statement requests comparison. Use the provided context and page description when present."
-                }
-            }
-        }
-    });
-    const answer = systemOneResponseSchema.parse(JSON.parse(result.content)).answers.result;
-    assert('noul' === answer.type, 'Invalid Jev boolean answer');
-    return {
-        value: answer.noul >= 0.5,
-        rawResponse: result.content,
-        usage: result.usage
-    };
-};
 const jevAdapters = {
     jev: {
-        supportedTasks: [
+        modelType: 'system-one',
+        supportedTaskTypes: [
             'locate',
             'boolean',
             'planning'
@@ -116,10 +12,10 @@ const jevAdapters = {
         supportedProtocols: [],
         locate: {
             kind: 'custom',
-            locateFn: locate
+            locateFn: jevLocate
         },
         insight: {
-            booleanFn: adapter_boolean
+            booleanFn: jevBoolean
         },
         planning: {
             kind: 'custom',

package/dist/es/ai-model/models/jev/adapter.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/jev/adapter.mjs","sources":["../../../../../src/ai-model/models/jev/adapter.ts"],"sourcesContent":["import { userPromptToString } from '@/common';\nimport type { TModelFamily } from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\nimport type { ModelAdapterDefinition } from '../../model-adapter/types';\nimport { callAI } from '../../service-caller';\nimport { systemOneResponseSchema } from '../../service-caller/system-one';\nimport { prepareUserPrompt } from '../../shared/multimodal-prompt';\nimport type { LocateFn } from '../../workflows/grounding/types';\nimport type { BooleanFn } from '../../workflows/insight/boolean';\nimport { mapJevBox, prepareJevImage } from './image';\nimport { jevPlan } from './planning';\n\nasync function prepareReferences(\n  prompt: Parameters<typeof prepareUserPrompt>[0],\n) {\n  const { referenceImages } = await prepareUserPrompt(prompt);\n  const prepared = await Promise.all(\n    referenceImages.map(async (image, index) => ({\n      key: `reference_${index}`,\n      name: image.name,\n      image: await prepareJevImage(image.url),\n    })),\n  );\n  return {\n    references: prepared.map(({ key, name }) => ({\n      image: `\\`${key}\\``,\n      name,\n    })),\n    images: Object.fromEntries(\n      prepared.map(({ key, image }) => [key, image.imageBase64]),\n    ),\n  };\n}\n\nconst locate: LocateFn = async ({\n  targetElementDescription,\n  locateImage,\n  options,\n}) => {\n  const image = await prepareJevImage(locateImage.imageBase64);\n  const references = await prepareReferences(targetElementDescription);\n  const target = userPromptToString(targetElementDescription);\n  const response = await callAI([], options.modelRuntime, {\n    abortSignal: options.abortSignal,\n    systemOneRequest: {\n      state: { target, references: references.references },\n      images: { screenshot: image.imageBase64, ...references.images },\n      questions: {\n        present: {\n          type: 'noul',\n          instructions:\n            'Is the target described in `target` present in `screenshot`? Reference images, if any, describe the target; they are not the screenshot.',\n        },\n        target: {\n          type: 'box',\n          instructions:\n            'The single target described in `target`, in `screenshot`.',\n        },\n      },\n    },\n  });\n  const { answers } = systemOneResponseSchema.parse(\n    JSON.parse(response.content),\n  );\n  assert(\n    answers.present.type === 'noul' && answers.target.type === 'box',\n    'Invalid Jev locate answers',\n  );\n  return {\n    rawResponse: response.content,\n    usage: response.usage,\n    ...(answers.present.noul >= 0.5\n      ? {\n          locatedPixelResult: mapJevBox(answers.target.box, image, locateImage),\n        }\n      : { errors: ['The target is not present in the screenshot'] }),\n  };\n};\n\nconst boolean: BooleanFn = async (options) => {\n  const references = await prepareReferences({\n    prompt: '',\n    ...options.multimodalPrompt,\n  });\n  const frames =\n    options.extractOption?.screenshotIncluded === false\n      ? []\n      : options.context.screenshotSequence?.length\n        ? options.context.screenshotSequence\n        : [options.context.screenshot];\n  const screenshots = await Promise.all(\n    frames.map((frame) => prepareJevImage(frame.base64)),\n  );\n  const result = await callAI([], options.modelRuntime, {\n    abortSignal: options.abortSignal,\n    systemOneRequest: {\n      state: {\n        statement: options.dataQuery,\n        context: options.extractOption?.context,\n        pageDescription: options.pageDescription,\n        frames: screenshots.map((_, index) => `\\`frame_${index}\\``),\n        references: references.references,\n      },\n      images: {\n        ...Object.fromEntries(\n          screenshots.map((image, index) => [\n            `frame_${index}`,\n            image.imageBase64,\n          ]),\n        ),\n        ...references.images,\n      },\n      questions: {\n        result: {\n          type: 'noul',\n          instructions:\n            'Is `statement` true? Screenshots in `frames` are ordered earliest to latest. For current state use the latest frame; for events use the relevant frames. Reference images are supporting context, not screenshots, unless the statement requests comparison. Use the provided context and page description when present.',\n        },\n      },\n    },\n  });\n  const answer = systemOneResponseSchema.parse(JSON.parse(result.content))\n    .answers.result;\n  assert(answer.type === 'noul', 'Invalid Jev boolean answer');\n  return {\n    value: answer.noul >= 0.5,\n    rawResponse: result.content,\n    usage: result.usage,\n  };\n};\n\nexport const jevAdapters = {\n  jev: {\n    supportedTasks: ['locate', 'boolean', 'planning'],\n    supportedProtocols: [],\n    locate: { kind: 'custom', locateFn: locate },\n    insight: { booleanFn: boolean },\n    planning: {\n      kind: 'custom',\n      planFn: jevPlan,\n      cacheEnabled: false,\n      supportsActionDeepLocate: false,\n      defaultReplanningCycleLimit: 20,\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'jev'>;\n"],"names":["prepareReferences","prompt","referenceImages","prepareUserPrompt","prepared","Promise","image","index","prepareJevImage","key","name","Object","locate","targetElementDescription","locateImage","options","references","target","userPromptToString","response","callAI","answers","systemOneResponseSchema","JSON","assert","mapJevBox","boolean","frames","screenshots","frame","result","_","answer","jevAdapters","jevPlan"],"mappings":";;;;;;;AAYA,eAAeA,kBACbC,MAA+C;IAE/C,MAAM,EAAEC,eAAe,EAAE,GAAG,MAAMC,kBAAkBF;IACpD,MAAMG,WAAW,MAAMC,QAAQ,GAAG,CAChCH,gBAAgB,GAAG,CAAC,OAAOI,OAAOC,QAAW;YAC3C,KAAK,CAAC,UAAU,EAAEA,OAAO;YACzB,MAAMD,MAAM,IAAI;YAChB,OAAO,MAAME,gBAAgBF,MAAM,GAAG;QACxC;IAEF,OAAO;QACL,YAAYF,SAAS,GAAG,CAAC,CAAC,EAAEK,GAAG,EAAEC,IAAI,EAAE,GAAM;gBAC3C,OAAO,CAAC,EAAE,EAAED,IAAI,EAAE,CAAC;gBACnBC;YACF;QACA,QAAQC,OAAO,WAAW,CACxBP,SAAS,GAAG,CAAC,CAAC,EAAEK,GAAG,EAAEH,KAAK,EAAE,GAAK;gBAACG;gBAAKH,MAAM,WAAW;aAAC;IAE7D;AACF;AAEA,MAAMM,SAAmB,OAAO,EAC9BC,wBAAwB,EACxBC,WAAW,EACXC,OAAO,EACR;IACC,MAAMT,QAAQ,MAAME,gBAAgBM,YAAY,WAAW;IAC3D,MAAME,aAAa,MAAMhB,kBAAkBa;IAC3C,MAAMI,SAASC,mBAAmBL;IAClC,MAAMM,WAAW,MAAMC,OAAO,EAAE,EAAEL,QAAQ,YAAY,EAAE;QACtD,aAAaA,QAAQ,WAAW;QAChC,kBAAkB;YAChB,OAAO;gBAAEE;gBAAQ,YAAYD,WAAW,UAAU;YAAC;YACnD,QAAQ;gBAAE,YAAYV,MAAM,WAAW;gBAAE,GAAGU,WAAW,MAAM;YAAC;YAC9D,WAAW;gBACT,SAAS;oBACP,MAAM;oBACN,cACE;gBACJ;gBACA,QAAQ;oBACN,MAAM;oBACN,cACE;gBACJ;YACF;QACF;IACF;IACA,MAAM,EAAEK,OAAO,EAAE,GAAGC,wBAAwB,KAAK,CAC/CC,KAAK,KAAK,CAACJ,SAAS,OAAO;IAE7BK,OACEH,AAAyB,WAAzBA,QAAQ,OAAO,CAAC,IAAI,IAAeA,AAAwB,UAAxBA,QAAQ,MAAM,CAAC,IAAI,EACtD;IAEF,OAAO;QACL,aAAaF,SAAS,OAAO;QAC7B,OAAOA,SAAS,KAAK;QACrB,GAAIE,QAAQ,OAAO,CAAC,IAAI,IAAI,MACxB;YACE,oBAAoBI,UAAUJ,QAAQ,MAAM,CAAC,GAAG,EAAEf,OAAOQ;QAC3D,IACA;YAAE,QAAQ;gBAAC;aAA8C;QAAC,CAAC;IACjE;AACF;AAEA,MAAMY,kBAAqB,OAAOX;IAChC,MAAMC,aAAa,MAAMhB,kBAAkB;QACzC,QAAQ;QACR,GAAGe,QAAQ,gBAAgB;IAC7B;IACA,MAAMY,SACJZ,QAAQ,aAAa,EAAE,uBAAuB,QAC1C,EAAE,GACFA,QAAQ,OAAO,CAAC,kBAAkB,EAAE,SAClCA,QAAQ,OAAO,CAAC,kBAAkB,GAClC;QAACA,QAAQ,OAAO,CAAC,UAAU;KAAC;IACpC,MAAMa,cAAc,MAAMvB,QAAQ,GAAG,CACnCsB,OAAO,GAAG,CAAC,CAACE,QAAUrB,gBAAgBqB,MAAM,MAAM;IAEpD,MAAMC,SAAS,MAAMV,OAAO,EAAE,EAAEL,QAAQ,YAAY,EAAE;QACpD,aAAaA,QAAQ,WAAW;QAChC,kBAAkB;YAChB,OAAO;gBACL,WAAWA,QAAQ,SAAS;gBAC5B,SAASA,QAAQ,aAAa,EAAE;gBAChC,iBAAiBA,QAAQ,eAAe;gBACxC,QAAQa,YAAY,GAAG,CAAC,CAACG,GAAGxB,QAAU,CAAC,QAAQ,EAAEA,MAAM,EAAE,CAAC;gBAC1D,YAAYS,WAAW,UAAU;YACnC;YACA,QAAQ;gBACN,GAAGL,OAAO,WAAW,CACnBiB,YAAY,GAAG,CAAC,CAACtB,OAAOC,QAAU;wBAChC,CAAC,MAAM,EAAEA,OAAO;wBAChBD,MAAM,WAAW;qBAClB,EACF;gBACD,GAAGU,WAAW,MAAM;YACtB;YACA,WAAW;gBACT,QAAQ;oBACN,MAAM;oBACN,cACE;gBACJ;YACF;QACF;IACF;IACA,MAAMgB,SAASV,wBAAwB,KAAK,CAACC,KAAK,KAAK,CAACO,OAAO,OAAO,GACnE,OAAO,CAAC,MAAM;IACjBN,OAAOQ,AAAgB,WAAhBA,OAAO,IAAI,EAAa;IAC/B,OAAO;QACL,OAAOA,OAAO,IAAI,IAAI;QACtB,aAAaF,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,MAAMG,cAAc;IACzB,KAAK;QACH,gBAAgB;YAAC;YAAU;YAAW;SAAW;QACjD,oBAAoB,EAAE;QACtB,QAAQ;YAAE,MAAM;YAAU,UAAUrB;QAAO;QAC3C,SAAS;YAAE,WAAWc;QAAQ;QAC9B,UAAU;YACR,MAAM;YACN,QAAQQ;YACR,cAAc;YACd,0BAA0B;YAC1B,6BAA6B;QAC/B;IACF;AACF"}
+{"version":3,"file":"ai-model/models/jev/adapter.mjs","sources":["../../../../../src/ai-model/models/jev/adapter.ts"],"sourcesContent":["import type { TModelFamily } from '@midscene/shared/env';\nimport type { ModelAdapterDefinition } from '../../model-adapter/types';\nimport { jevBoolean } from './boolean';\nimport { jevLocate } from './locate';\nimport { jevPlan } from './planning';\n\nexport const jevAdapters = {\n  jev: {\n    modelType: 'system-one',\n    supportedTaskTypes: ['locate', 'boolean', 'planning'],\n    supportedProtocols: [],\n    locate: { kind: 'custom', locateFn: jevLocate },\n    insight: { booleanFn: jevBoolean },\n    planning: {\n      kind: 'custom',\n      planFn: jevPlan,\n      cacheEnabled: false,\n      supportsActionDeepLocate: false,\n      defaultReplanningCycleLimit: 20,\n    },\n  },\n} satisfies Pick<Record<TModelFamily, ModelAdapterDefinition>, 'jev'>;\n"],"names":["jevAdapters","jevLocate","jevBoolean","jevPlan"],"mappings":";;;AAMO,MAAMA,cAAc;IACzB,KAAK;QACH,WAAW;QACX,oBAAoB;YAAC;YAAU;YAAW;SAAW;QACrD,oBAAoB,EAAE;QACtB,QAAQ;YAAE,MAAM;YAAU,UAAUC;QAAU;QAC9C,SAAS;YAAE,WAAWC;QAAW;QACjC,UAAU;YACR,MAAM;YACN,QAAQC;YACR,cAAc;YACd,0BAA0B;YAC1B,6BAA6B;QAC/B;IACF;AACF"}

package/dist/es/ai-model/models/jev/boolean.mjs

@@ -0,0 +1,48 @@
+import { assert } from "@midscene/shared/utils";
+import { callSystemOneAI } from "../../service-caller/index.mjs";
+import { systemOneResponseSchema } from "../../service-caller/system-one/schema.mjs";
+import { prepareJevImage, prepareReferences } from "./image.mjs";
+const jevBoolean = async (options)=>{
+    const references = await prepareReferences({
+        prompt: '',
+        ...options.multimodalPrompt
+    });
+    const frames = options.extractOption?.screenshotIncluded === false ? [] : options.context.screenshotSequence?.length ? options.context.screenshotSequence : [
+        options.context.screenshot
+    ];
+    const screenshots = await Promise.all(frames.map((frame)=>prepareJevImage(frame.base64)));
+    const result = await callSystemOneAI({
+        state: {
+            statement: options.dataQuery,
+            context: options.extractOption?.context,
+            pageDescription: options.pageDescription,
+            frames: screenshots.map((_, index)=>`\`frame_${index}\``),
+            references: references.references
+        },
+        images: {
+            ...Object.fromEntries(screenshots.map((image, index)=>[
+                    `frame_${index}`,
+                    image.imageBase64
+                ])),
+            ...references.images
+        },
+        questions: {
+            result: {
+                type: 'noul',
+                instructions: "Is `statement` true? Screenshots in `frames` are ordered earliest to latest. For current state use the latest frame; for events use the relevant frames. Reference images are supporting context, not screenshots, unless the statement requests comparison. Use the provided context and page description when present."
+            }
+        }
+    }, options.modelRuntime, {
+        abortSignal: options.abortSignal
+    });
+    const answer = systemOneResponseSchema.parse(JSON.parse(result.content)).answers.result;
+    assert('noul' === answer.type, 'Invalid Jev boolean answer');
+    return {
+        value: answer.noul >= 0.5,
+        rawResponse: result.content,
+        usage: result.usage
+    };
+};
+export { jevBoolean };
+
+//# sourceMappingURL=boolean.mjs.map

package/dist/es/ai-model/models/jev/boolean.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/jev/boolean.mjs","sources":["../../../../../src/ai-model/models/jev/boolean.ts"],"sourcesContent":["import { assert } from '@midscene/shared/utils';\nimport { callSystemOneAI } from '../../service-caller';\nimport { systemOneResponseSchema } from '../../service-caller/system-one/schema';\nimport type { BooleanFn } from '../../workflows/insight/boolean';\nimport { prepareJevImage, prepareReferences } from './image';\n\nexport const jevBoolean: BooleanFn = async (options) => {\n  const references = await prepareReferences({\n    prompt: '',\n    ...options.multimodalPrompt,\n  });\n  const frames =\n    options.extractOption?.screenshotIncluded === false\n      ? []\n      : options.context.screenshotSequence?.length\n        ? options.context.screenshotSequence\n        : [options.context.screenshot];\n  const screenshots = await Promise.all(\n    frames.map((frame) => prepareJevImage(frame.base64)),\n  );\n  const result = await callSystemOneAI(\n    {\n      state: {\n        statement: options.dataQuery,\n        context: options.extractOption?.context,\n        pageDescription: options.pageDescription,\n        frames: screenshots.map((_, index) => `\\`frame_${index}\\``),\n        references: references.references,\n      },\n      images: {\n        ...Object.fromEntries(\n          screenshots.map((image, index) => [\n            `frame_${index}`,\n            image.imageBase64,\n          ]),\n        ),\n        ...references.images,\n      },\n      questions: {\n        result: {\n          type: 'noul',\n          instructions:\n            'Is `statement` true? Screenshots in `frames` are ordered earliest to latest. For current state use the latest frame; for events use the relevant frames. Reference images are supporting context, not screenshots, unless the statement requests comparison. Use the provided context and page description when present.',\n        },\n      },\n    },\n    options.modelRuntime,\n    { abortSignal: options.abortSignal },\n  );\n  const answer = systemOneResponseSchema.parse(JSON.parse(result.content))\n    .answers.result;\n  assert(answer.type === 'noul', 'Invalid Jev boolean answer');\n  return {\n    value: answer.noul >= 0.5,\n    rawResponse: result.content,\n    usage: result.usage,\n  };\n};\n"],"names":["jevBoolean","options","references","prepareReferences","frames","screenshots","Promise","frame","prepareJevImage","result","callSystemOneAI","_","index","Object","image","answer","systemOneResponseSchema","JSON","assert"],"mappings":";;;;AAMO,MAAMA,aAAwB,OAAOC;IAC1C,MAAMC,aAAa,MAAMC,kBAAkB;QACzC,QAAQ;QACR,GAAGF,QAAQ,gBAAgB;IAC7B;IACA,MAAMG,SACJH,QAAQ,aAAa,EAAE,uBAAuB,QAC1C,EAAE,GACFA,QAAQ,OAAO,CAAC,kBAAkB,EAAE,SAClCA,QAAQ,OAAO,CAAC,kBAAkB,GAClC;QAACA,QAAQ,OAAO,CAAC,UAAU;KAAC;IACpC,MAAMI,cAAc,MAAMC,QAAQ,GAAG,CACnCF,OAAO,GAAG,CAAC,CAACG,QAAUC,gBAAgBD,MAAM,MAAM;IAEpD,MAAME,SAAS,MAAMC,gBACnB;QACE,OAAO;YACL,WAAWT,QAAQ,SAAS;YAC5B,SAASA,QAAQ,aAAa,EAAE;YAChC,iBAAiBA,QAAQ,eAAe;YACxC,QAAQI,YAAY,GAAG,CAAC,CAACM,GAAGC,QAAU,CAAC,QAAQ,EAAEA,MAAM,EAAE,CAAC;YAC1D,YAAYV,WAAW,UAAU;QACnC;QACA,QAAQ;YACN,GAAGW,OAAO,WAAW,CACnBR,YAAY,GAAG,CAAC,CAACS,OAAOF,QAAU;oBAChC,CAAC,MAAM,EAAEA,OAAO;oBAChBE,MAAM,WAAW;iBAClB,EACF;YACD,GAAGZ,WAAW,MAAM;QACtB;QACA,WAAW;YACT,QAAQ;gBACN,MAAM;gBACN,cACE;YACJ;QACF;IACF,GACAD,QAAQ,YAAY,EACpB;QAAE,aAAaA,QAAQ,WAAW;IAAC;IAErC,MAAMc,SAASC,wBAAwB,KAAK,CAACC,KAAK,KAAK,CAACR,OAAO,OAAO,GACnE,OAAO,CAAC,MAAM;IACjBS,OAAOH,AAAgB,WAAhBA,OAAO,IAAI,EAAa;IAC/B,OAAO;QACL,OAAOA,OAAO,IAAI,IAAI;QACtB,aAAaN,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/models/jev/image.mjs

@@ -1,48 +1,51 @@
 import { Buffer } from "node:buffer";
-import { imageInfoOfBase64, padBase64ImageToMinimumSize, parseBase64, preProcessImageUrl, resizeBase64ImageToJpeg } from "@midscene/shared/img";
+import { parseBase64, preProcessImageUrl } from "@midscene/shared/img";
 import { assert } from "@midscene/shared/utils";
+import { createLocateResultCodec } from "../../shared/model-locate-result/index.mjs";
+import { prepareUserPrompt } from "../../shared/multimodal-prompt.mjs";
 async function prepareJevImage(url) {
     const base64 = await preProcessImageUrl(url, true);
-    const source = await imageInfoOfBase64(base64);
-    const scale = Math.min(1, 1568 / source.width, 1568 / source.height);
-    const contentSize = {
-        width: Math.max(1, Math.round(source.width * scale)),
-        height: Math.max(1, Math.round(source.height * scale))
-    };
-    const resized = await resizeBase64ImageToJpeg(base64, {
-        targetSize: contentSize
-    });
-    const prepared = await padBase64ImageToMinimumSize(resized, 280);
-    assert(Buffer.from(parseBase64(prepared.imageBase64).body, 'base64').length <= 16777216, 'Jev image exceeds 16 MiB');
+    assert(Buffer.from(parseBase64(base64).body, 'base64').length <= 16777216, 'Jev image exceeds 16 MiB');
     return {
-        ...prepared,
-        contentSize,
-        sourceSize: {
-            width: source.width,
-            height: source.height
-        }
+        imageBase64: base64
     };
 }
-function mapJevBox(box, image, targetSize) {
-    const left = box.left * image.width * targetSize.width / image.contentSize.width;
-    const top = box.top * image.height * targetSize.height / image.contentSize.height;
-    const right = box.right * image.width * targetSize.width / image.contentSize.width;
-    const bottom = box.bottom * image.height * targetSize.height / image.contentSize.height;
-    const center = [
-        (left + right) / 2,
-        (top + bottom) / 2
-    ];
-    assert(center[0] >= 0 && center[0] < targetSize.width && center[1] >= 0 && center[1] < targetSize.height, 'Jev returned a target in image padding, outside the screenshot');
+const jevBoxCodec = createLocateResultCodec({
+    coordinates: {
+        shape: 'bbox',
+        normalizedBy: 1,
+        rounding: 'none',
+        boundsType: 'edges'
+    }
+});
+function mapJevBox(box, targetSize) {
+    return jevBoxCodec.toPixelResult([
+        box.left,
+        box.top,
+        box.right,
+        box.bottom
+    ], {
+        preparedSize: targetSize
+    });
+}
+async function prepareReferences(prompt) {
+    const { referenceImages } = await prepareUserPrompt(prompt);
+    const prepared = await Promise.all(referenceImages.map(async (image, index)=>({
+            key: `reference_${index}`,
+            name: image.name,
+            image: await prepareJevImage(image.url)
+        })));
     return {
-        center,
-        rect: {
-            left,
-            top,
-            width: right - left,
-            height: bottom - top
-        }
+        references: prepared.map(({ key, name })=>({
+                image: `\`${key}\``,
+                name
+            })),
+        images: Object.fromEntries(prepared.map(({ key, image })=>[
+                key,
+                image.imageBase64
+            ]))
     };
 }
-export { mapJevBox, prepareJevImage };
+export { mapJevBox, prepareJevImage, prepareReferences };
 
 //# sourceMappingURL=image.mjs.map

package/dist/es/ai-model/models/jev/image.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/models/jev/image.mjs","sources":["../../../../../src/ai-model/models/jev/image.ts"],"sourcesContent":["import { Buffer } from 'node:buffer';\nimport {\n  imageInfoOfBase64,\n  padBase64ImageToMinimumSize,\n  parseBase64,\n  preProcessImageUrl,\n  resizeBase64ImageToJpeg,\n} from '@midscene/shared/img';\nimport { assert } from '@midscene/shared/utils';\nimport type { PixelLocateResult } from '../../shared/model-locate-result';\n\nexport async function prepareJevImage(url: string) {\n  const base64 = await preProcessImageUrl(url, true);\n  const source = await imageInfoOfBase64(base64);\n  const scale = Math.min(1, 1568 / source.width, 1568 / source.height);\n  const contentSize = {\n    width: Math.max(1, Math.round(source.width * scale)),\n    height: Math.max(1, Math.round(source.height * scale)),\n  };\n  const resized = await resizeBase64ImageToJpeg(base64, {\n    targetSize: contentSize,\n  });\n  const prepared = await padBase64ImageToMinimumSize(resized, 280);\n  assert(\n    Buffer.from(parseBase64(prepared.imageBase64).body, 'base64').length <=\n      16 * 1024 * 1024,\n    'Jev image exceeds 16 MiB',\n  );\n  return {\n    ...prepared,\n    contentSize,\n    sourceSize: { width: source.width, height: source.height },\n  };\n}\n\nexport function mapJevBox(\n  box: { left: number; top: number; right: number; bottom: number },\n  image: Awaited<ReturnType<typeof prepareJevImage>>,\n  targetSize: { width: number; height: number },\n): PixelLocateResult {\n  // Jev returns fractions of the padded image. Undo padding and scale before\n  // the grounding workflow maps a DeepLocate crop back to the full screenshot.\n  const left =\n    (box.left * image.width * targetSize.width) / image.contentSize.width;\n  const top =\n    (box.top * image.height * targetSize.height) / image.contentSize.height;\n  const right =\n    (box.right * image.width * targetSize.width) / image.contentSize.width;\n  const bottom =\n    (box.bottom * image.height * targetSize.height) / image.contentSize.height;\n  const center: [number, number] = [(left + right) / 2, (top + bottom) / 2];\n  assert(\n    center[0] >= 0 &&\n      center[0] < targetSize.width &&\n      center[1] >= 0 &&\n      center[1] < targetSize.height,\n    'Jev returned a target in image padding, outside the screenshot',\n  );\n  return {\n    center,\n    rect: { left, top, width: right - left, height: bottom - top },\n  };\n}\n"],"names":["prepareJevImage","url","base64","preProcessImageUrl","source","imageInfoOfBase64","scale","Math","contentSize","resized","resizeBase64ImageToJpeg","prepared","padBase64ImageToMinimumSize","assert","Buffer","parseBase64","mapJevBox","box","image","targetSize","left","top","right","bottom","center"],"mappings":";;;AAWO,eAAeA,gBAAgBC,GAAW;IAC/C,MAAMC,SAAS,MAAMC,mBAAmBF,KAAK;IAC7C,MAAMG,SAAS,MAAMC,kBAAkBH;IACvC,MAAMI,QAAQC,KAAK,GAAG,CAAC,GAAG,OAAOH,OAAO,KAAK,EAAE,OAAOA,OAAO,MAAM;IACnE,MAAMI,cAAc;QAClB,OAAOD,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACH,OAAO,KAAK,GAAGE;QAC7C,QAAQC,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACH,OAAO,MAAM,GAAGE;IACjD;IACA,MAAMG,UAAU,MAAMC,wBAAwBR,QAAQ;QACpD,YAAYM;IACd;IACA,MAAMG,WAAW,MAAMC,4BAA4BH,SAAS;IAC5DI,OACEC,OAAO,IAAI,CAACC,YAAYJ,SAAS,WAAW,EAAE,IAAI,EAAE,UAAU,MAAM,IAClE,UACF;IAEF,OAAO;QACL,GAAGA,QAAQ;QACXH;QACA,YAAY;YAAE,OAAOJ,OAAO,KAAK;YAAE,QAAQA,OAAO,MAAM;QAAC;IAC3D;AACF;AAEO,SAASY,UACdC,GAAiE,EACjEC,KAAkD,EAClDC,UAA6C;IAI7C,MAAMC,OACHH,IAAI,IAAI,GAAGC,MAAM,KAAK,GAAGC,WAAW,KAAK,GAAID,MAAM,WAAW,CAAC,KAAK;IACvE,MAAMG,MACHJ,IAAI,GAAG,GAAGC,MAAM,MAAM,GAAGC,WAAW,MAAM,GAAID,MAAM,WAAW,CAAC,MAAM;IACzE,MAAMI,QACHL,IAAI,KAAK,GAAGC,MAAM,KAAK,GAAGC,WAAW,KAAK,GAAID,MAAM,WAAW,CAAC,KAAK;IACxE,MAAMK,SACHN,IAAI,MAAM,GAAGC,MAAM,MAAM,GAAGC,WAAW,MAAM,GAAID,MAAM,WAAW,CAAC,MAAM;IAC5E,MAAMM,SAA2B;QAAEJ,CAAAA,OAAOE,KAAI,IAAK;QAAID,CAAAA,MAAME,MAAK,IAAK;KAAE;IACzEV,OACEW,MAAM,CAAC,EAAE,IAAI,KACXA,MAAM,CAAC,EAAE,GAAGL,WAAW,KAAK,IAC5BK,MAAM,CAAC,EAAE,IAAI,KACbA,MAAM,CAAC,EAAE,GAAGL,WAAW,MAAM,EAC/B;IAEF,OAAO;QACLK;QACA,MAAM;YAAEJ;YAAMC;YAAK,OAAOC,QAAQF;YAAM,QAAQG,SAASF;QAAI;IAC/D;AACF"}
+{"version":3,"file":"ai-model/models/jev/image.mjs","sources":["../../../../../src/ai-model/models/jev/image.ts"],"sourcesContent":["import { Buffer } from 'node:buffer';\nimport { parseBase64, preProcessImageUrl } from '@midscene/shared/img';\nimport { assert } from '@midscene/shared/utils';\nimport { createLocateResultCodec } from '../../shared/model-locate-result';\nimport type { PixelLocateResult } from '../../shared/model-locate-result';\nimport { prepareUserPrompt } from '../../shared/multimodal-prompt';\n\nexport async function prepareJevImage(url: string) {\n  const base64 = await preProcessImageUrl(url, true);\n  assert(\n    Buffer.from(parseBase64(base64).body, 'base64').length <= 16 * 1024 * 1024,\n    'Jev image exceeds 16 MiB',\n  );\n  return { imageBase64: base64 };\n}\n\nconst jevBoxCodec = createLocateResultCodec({\n  coordinates: {\n    shape: 'bbox',\n    normalizedBy: 1,\n    rounding: 'none',\n    boundsType: 'edges',\n  },\n});\n\nexport function mapJevBox(\n  box: { left: number; top: number; right: number; bottom: number },\n  targetSize: { width: number; height: number },\n): PixelLocateResult {\n  return jevBoxCodec.toPixelResult([box.left, box.top, box.right, box.bottom], {\n    preparedSize: targetSize,\n  });\n}\n\nexport async function prepareReferences(\n  prompt: Parameters<typeof prepareUserPrompt>[0],\n) {\n  const { referenceImages } = await prepareUserPrompt(prompt);\n  const prepared = await Promise.all(\n    referenceImages.map(async (image, index) => ({\n      key: `reference_${index}`,\n      name: image.name,\n      image: await prepareJevImage(image.url),\n    })),\n  );\n  return {\n    references: prepared.map(({ key, name }) => ({\n      image: `\\`${key}\\``,\n      name,\n    })),\n    images: Object.fromEntries(\n      prepared.map(({ key, image }) => [key, image.imageBase64]),\n    ),\n  };\n}\n"],"names":["prepareJevImage","url","base64","preProcessImageUrl","assert","Buffer","parseBase64","jevBoxCodec","createLocateResultCodec","mapJevBox","box","targetSize","prepareReferences","prompt","referenceImages","prepareUserPrompt","prepared","Promise","image","index","key","name","Object"],"mappings":";;;;;AAOO,eAAeA,gBAAgBC,GAAW;IAC/C,MAAMC,SAAS,MAAMC,mBAAmBF,KAAK;IAC7CG,OACEC,OAAO,IAAI,CAACC,YAAYJ,QAAQ,IAAI,EAAE,UAAU,MAAM,IAAI,UAC1D;IAEF,OAAO;QAAE,aAAaA;IAAO;AAC/B;AAEA,MAAMK,cAAcC,wBAAwB;IAC1C,aAAa;QACX,OAAO;QACP,cAAc;QACd,UAAU;QACV,YAAY;IACd;AACF;AAEO,SAASC,UACdC,GAAiE,EACjEC,UAA6C;IAE7C,OAAOJ,YAAY,aAAa,CAAC;QAACG,IAAI,IAAI;QAAEA,IAAI,GAAG;QAAEA,IAAI,KAAK;QAAEA,IAAI,MAAM;KAAC,EAAE;QAC3E,cAAcC;IAChB;AACF;AAEO,eAAeC,kBACpBC,MAA+C;IAE/C,MAAM,EAAEC,eAAe,EAAE,GAAG,MAAMC,kBAAkBF;IACpD,MAAMG,WAAW,MAAMC,QAAQ,GAAG,CAChCH,gBAAgB,GAAG,CAAC,OAAOI,OAAOC,QAAW;YAC3C,KAAK,CAAC,UAAU,EAAEA,OAAO;YACzB,MAAMD,MAAM,IAAI;YAChB,OAAO,MAAMlB,gBAAgBkB,MAAM,GAAG;QACxC;IAEF,OAAO;QACL,YAAYF,SAAS,GAAG,CAAC,CAAC,EAAEI,GAAG,EAAEC,IAAI,EAAE,GAAM;gBAC3C,OAAO,CAAC,EAAE,EAAED,IAAI,EAAE,CAAC;gBACnBC;YACF;QACA,QAAQC,OAAO,WAAW,CACxBN,SAAS,GAAG,CAAC,CAAC,EAAEI,GAAG,EAAEF,KAAK,EAAE,GAAK;gBAACE;gBAAKF,MAAM,WAAW;aAAC;IAE7D;AACF"}

package/dist/es/ai-model/models/jev/locate.mjs

@@ -0,0 +1,48 @@
+import { userPromptToString } from "../../../common.mjs";
+import { assert } from "@midscene/shared/utils";
+import { callSystemOneAI } from "../../service-caller/index.mjs";
+import { systemOneResponseSchema } from "../../service-caller/system-one/schema.mjs";
+import { mapJevBox, prepareJevImage, prepareReferences } from "./image.mjs";
+const jevLocate = async ({ targetElementDescription, locateImage, options })=>{
+    const image = await prepareJevImage(locateImage.imageBase64);
+    const references = await prepareReferences(targetElementDescription);
+    const target = userPromptToString(targetElementDescription);
+    const response = await callSystemOneAI({
+        state: {
+            target,
+            references: references.references
+        },
+        images: {
+            screenshot: image.imageBase64,
+            ...references.images
+        },
+        questions: {
+            present: {
+                type: 'noul',
+                instructions: 'Is the target described in `target` present in `screenshot`? Reference images, if any, describe the target; they are not the screenshot.'
+            },
+            target: {
+                type: 'box',
+                instructions: 'The single target described in `target`, in `screenshot`.'
+            }
+        }
+    }, options.modelRuntime, {
+        abortSignal: options.abortSignal
+    });
+    const { answers } = systemOneResponseSchema.parse(JSON.parse(response.content));
+    assert('noul' === answers.present.type && 'box' === answers.target.type, 'Invalid Jev locate answers');
+    return {
+        rawResponse: response.content,
+        usage: response.usage,
+        ...answers.present.noul >= 0.5 ? {
+            locatedPixelResult: mapJevBox(answers.target.box, locateImage)
+        } : {
+            errors: [
+                'The target is not present in the screenshot'
+            ]
+        }
+    };
+};
+export { jevLocate };
+
+//# sourceMappingURL=locate.mjs.map

package/dist/es/ai-model/models/jev/locate.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/jev/locate.mjs","sources":["../../../../../src/ai-model/models/jev/locate.ts"],"sourcesContent":["import { userPromptToString } from '@/common';\nimport { assert } from '@midscene/shared/utils';\nimport { callSystemOneAI } from '../../service-caller';\nimport { systemOneResponseSchema } from '../../service-caller/system-one/schema';\nimport type { LocateFn } from '../../workflows/grounding/types';\nimport { mapJevBox, prepareJevImage, prepareReferences } from './image';\n\nexport const jevLocate: LocateFn = async ({\n  targetElementDescription,\n  locateImage,\n  options,\n}) => {\n  const image = await prepareJevImage(locateImage.imageBase64);\n  const references = await prepareReferences(targetElementDescription);\n  const target = userPromptToString(targetElementDescription);\n  const response = await callSystemOneAI(\n    {\n      state: { target, references: references.references },\n      images: { screenshot: image.imageBase64, ...references.images },\n      questions: {\n        present: {\n          type: 'noul',\n          instructions:\n            'Is the target described in `target` present in `screenshot`? Reference images, if any, describe the target; they are not the screenshot.',\n        },\n        target: {\n          type: 'box',\n          instructions:\n            'The single target described in `target`, in `screenshot`.',\n        },\n      },\n    },\n    options.modelRuntime,\n    { abortSignal: options.abortSignal },\n  );\n  const { answers } = systemOneResponseSchema.parse(\n    JSON.parse(response.content),\n  );\n  assert(\n    answers.present.type === 'noul' && answers.target.type === 'box',\n    'Invalid Jev locate answers',\n  );\n  return {\n    rawResponse: response.content,\n    usage: response.usage,\n    ...(answers.present.noul >= 0.5\n      ? {\n          locatedPixelResult: mapJevBox(answers.target.box, locateImage),\n        }\n      : { errors: ['The target is not present in the screenshot'] }),\n  };\n};\n"],"names":["jevLocate","targetElementDescription","locateImage","options","image","prepareJevImage","references","prepareReferences","target","userPromptToString","response","callSystemOneAI","answers","systemOneResponseSchema","JSON","assert","mapJevBox"],"mappings":";;;;;AAOO,MAAMA,YAAsB,OAAO,EACxCC,wBAAwB,EACxBC,WAAW,EACXC,OAAO,EACR;IACC,MAAMC,QAAQ,MAAMC,gBAAgBH,YAAY,WAAW;IAC3D,MAAMI,aAAa,MAAMC,kBAAkBN;IAC3C,MAAMO,SAASC,mBAAmBR;IAClC,MAAMS,WAAW,MAAMC,gBACrB;QACE,OAAO;YAAEH;YAAQ,YAAYF,WAAW,UAAU;QAAC;QACnD,QAAQ;YAAE,YAAYF,MAAM,WAAW;YAAE,GAAGE,WAAW,MAAM;QAAC;QAC9D,WAAW;YACT,SAAS;gBACP,MAAM;gBACN,cACE;YACJ;YACA,QAAQ;gBACN,MAAM;gBACN,cACE;YACJ;QACF;IACF,GACAH,QAAQ,YAAY,EACpB;QAAE,aAAaA,QAAQ,WAAW;IAAC;IAErC,MAAM,EAAES,OAAO,EAAE,GAAGC,wBAAwB,KAAK,CAC/CC,KAAK,KAAK,CAACJ,SAAS,OAAO;IAE7BK,OACEH,AAAyB,WAAzBA,QAAQ,OAAO,CAAC,IAAI,IAAeA,AAAwB,UAAxBA,QAAQ,MAAM,CAAC,IAAI,EACtD;IAEF,OAAO;QACL,aAAaF,SAAS,OAAO;QAC7B,OAAOA,SAAS,KAAK;QACrB,GAAIE,QAAQ,OAAO,CAAC,IAAI,IAAI,MACxB;YACE,oBAAoBI,UAAUJ,QAAQ,MAAM,CAAC,GAAG,EAAEV;QACpD,IACA;YAAE,QAAQ;gBAAC;aAA8C;QAAC,CAAC;IACjE;AACF"}

package/dist/es/ai-model/models/jev/planning-action.mjs

@@ -0,0 +1,35 @@
+import { findAllMidsceneLocatorField, parseActionParam } from "../../../common.mjs";
+import { assert } from "@midscene/shared/utils";
+import { mapJevBox } from "./image.mjs";
+function buildJevAction(decision, context, instruction, options) {
+    const { choice } = decision;
+    if ('complete' === choice.kind) return [];
+    assert('action' === choice.kind, 'Delegated actions must use the default planner');
+    const action = choice.action;
+    const fields = action.paramSchema ? findAllMidsceneLocatorField(action.paramSchema) : [];
+    const locatedPixelResult = (()=>{
+        if (!options.includeLocateInPlanning || 1 !== fields.length) return;
+        assert(decision.target?.type === 'box', 'Missing Jev planning target box');
+        return mapJevBox(decision.target.box, options.context.shotSize);
+    })();
+    const param = Object.fromEntries(fields.map((field)=>[
+            field,
+            {
+                prompt: `Locate the target for parameter ${JSON.stringify(field)} of action ${JSON.stringify(action.name)} (${action.description ?? action.name}) for the NEXT step toward this goal. Use the current screenshot as authoritative and consider execution feedback. Task context: ${JSON.stringify(context.state)}`,
+                images: instruction.referenceImages,
+                ...locatedPixelResult ? {
+                    locatedPixelResult
+                } : {}
+            }
+        ]));
+    parseActionParam(param, action.paramSchema);
+    return [
+        {
+            type: action.name,
+            param
+        }
+    ];
+}
+export { buildJevAction };
+
+//# sourceMappingURL=planning-action.mjs.map

package/dist/es/ai-model/models/jev/planning-action.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/models/jev/planning-action.mjs","sources":["../../../../../src/ai-model/models/jev/planning-action.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/common';\nimport type { PlanningAction } from '@/types';\nimport { assert } from '@midscene/shared/utils';\nimport type { PlanFn, PlanOptions } from '../../workflows/planning/types';\nimport { mapJevBox } from './image';\nimport type { JevPlanningContext } from './planning-context';\nimport type { JevActionDecision } from './planning-decision';\n\nexport function buildJevAction(\n  decision: JevActionDecision,\n  context: JevPlanningContext,\n  instruction: Parameters<PlanFn>[0],\n  options: PlanOptions,\n): PlanningAction[] {\n  const { choice } = decision;\n  if (choice.kind === 'complete') {\n    return [];\n  }\n  assert(\n    choice.kind === 'action',\n    'Delegated actions must use the default planner',\n  );\n  const action = choice.action;\n  const fields = action.paramSchema\n    ? findAllMidsceneLocatorField(action.paramSchema)\n    : [];\n  const locatedPixelResult = (() => {\n    if (!options.includeLocateInPlanning || fields.length !== 1) {\n      return undefined;\n    }\n    assert(decision.target?.type === 'box', 'Missing Jev planning target box');\n    return mapJevBox(decision.target.box, options.context.shotSize);\n  })();\n  const param = Object.fromEntries(\n    fields.map((field) => [\n      field,\n      {\n        prompt: `Locate the target for parameter ${JSON.stringify(field)} of action ${JSON.stringify(action.name)} (${action.description ?? action.name}) for the NEXT step toward this goal. Use the current screenshot as authoritative and consider execution feedback. Task context: ${JSON.stringify(context.state)}`,\n        images: instruction.referenceImages,\n        ...(locatedPixelResult ? { locatedPixelResult } : {}),\n      },\n    ]),\n  );\n  // Validate declared direct-action support before handing the plan to execution.\n  parseActionParam(param, action.paramSchema);\n  return [{ type: action.name, param }];\n}\n"],"names":["buildJevAction","decision","context","instruction","options","choice","assert","action","fields","findAllMidsceneLocatorField","locatedPixelResult","mapJevBox","param","Object","field","JSON","parseActionParam"],"mappings":";;;AAQO,SAASA,eACdC,QAA2B,EAC3BC,OAA2B,EAC3BC,WAAkC,EAClCC,OAAoB;IAEpB,MAAM,EAAEC,MAAM,EAAE,GAAGJ;IACnB,IAAII,AAAgB,eAAhBA,OAAO,IAAI,EACb,OAAO,EAAE;IAEXC,OACED,AAAgB,aAAhBA,OAAO,IAAI,EACX;IAEF,MAAME,SAASF,OAAO,MAAM;IAC5B,MAAMG,SAASD,OAAO,WAAW,GAC7BE,4BAA4BF,OAAO,WAAW,IAC9C,EAAE;IACN,MAAMG,qBAAsB,AAAC;QAC3B,IAAI,CAACN,QAAQ,uBAAuB,IAAII,AAAkB,MAAlBA,OAAO,MAAM,EACnD;QAEFF,OAAOL,SAAS,MAAM,EAAE,SAAS,OAAO;QACxC,OAAOU,UAAUV,SAAS,MAAM,CAAC,GAAG,EAAEG,QAAQ,OAAO,CAAC,QAAQ;IAChE;IACA,MAAMQ,QAAQC,OAAO,WAAW,CAC9BL,OAAO,GAAG,CAAC,CAACM,QAAU;YACpBA;YACA;gBACE,QAAQ,CAAC,gCAAgC,EAAEC,KAAK,SAAS,CAACD,OAAO,WAAW,EAAEC,KAAK,SAAS,CAACR,OAAO,IAAI,EAAE,EAAE,EAAEA,OAAO,WAAW,IAAIA,OAAO,IAAI,CAAC,iIAAiI,EAAEQ,KAAK,SAAS,CAACb,QAAQ,KAAK,GAAG;gBAClT,QAAQC,YAAY,eAAe;gBACnC,GAAIO,qBAAqB;oBAAEA;gBAAmB,IAAI,CAAC,CAAC;YACtD;SACD;IAGHM,iBAAiBJ,OAAOL,OAAO,WAAW;IAC1C,OAAO;QAAC;YAAE,MAAMA,OAAO,IAAI;YAAEK;QAAM;KAAE;AACvC"}

package/dist/es/ai-model/models/jev/planning-context.mjs

@@ -0,0 +1,38 @@
+import { prepareJevImage } from "./image.mjs";
+async function prepareJevPlanningContext(instruction, options) {
+    const history = options.conversationHistory;
+    const image = await prepareJevImage(options.context.screenshot.base64);
+    const references = await Promise.all(instruction.referenceImages.map(async (reference, index)=>({
+            key: `reference_${index}`,
+            name: reference.name,
+            image: await prepareJevImage(reference.url)
+        })));
+    const state = {
+        goal: instruction.text,
+        context: options.actionContext,
+        history: history.historicalLogsToText(),
+        memory: history.memoriesToText(),
+        feedback: history.pendingFeedbackMessage,
+        references: references.map(({ key, name })=>({
+                image: `\`${key}\``,
+                name
+            })),
+        availableActions: options.actionSpace.map(({ name, description })=>({
+                name,
+                description
+            }))
+    };
+    return {
+        state,
+        images: {
+            screenshot: image.imageBase64,
+            ...Object.fromEntries(references.map(({ key, image })=>[
+                    key,
+                    image.imageBase64
+                ]))
+        }
+    };
+}
+export { prepareJevPlanningContext };
+
+//# sourceMappingURL=planning-context.mjs.map