@midscene/core 1.15.5 → 1.15.6

package/dist/es/ai-model/service-caller/call-ai.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/call-ai.mjs","sources":["../../../../src/ai-model/service-caller/call-ai.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert, uuid } from '@midscene/shared/utils';\nimport type { ModelRuntime } from '../models';\nimport { callCodex, prepareCodexCall } from './codex/call-codex';\nimport { isCodexAppServerProvider } from './codex/codex-app-server';\nimport {\n  isModelCallRecordingEnabled,\n  recordModelCallEvent,\n} from './model-call-recorder';\nimport { type PreparedOpenAIInput, callOpenAI } from './openai/call-openai';\nimport { prepareChatCompletion } from './openai/chat-completion/chat-completion';\nimport { prepareResponses } from './openai/responses/responses';\nimport {\n  buildRequestAbortSignal,\n  resolveEffectiveTimeoutMs,\n  runWithAbortSignal,\n  waitForRetry,\n} from './request-timeout';\nimport type { ModelCallMessages } from './types';\nimport type {\n  AICallResult,\n  CallAIOptions,\n  ModelCallContext,\n  ModelCallResult,\n} from './types';\nimport {\n  appendAIRequestFailureSummary,\n  buildUsageInfo,\n  nextInternalCallId,\n  normalizeRetryCount,\n  toError,\n} from './utils';\n\nconst DEFAULT_MODEL_PROTOCOL = 'openai-chat';\n\nexport async function callAI(\n  messages: ModelCallMessages,\n  modelRuntime: ModelRuntime,\n  options?: CallAIOptions,\n): Promise<AICallResult> {\n  options?.abortSignal?.throwIfAborted();\n  const isStreaming = options?.stream === true;\n  if (isStreaming) {\n    // Explicit streaming is currently used only by recorder code generation.\n    // Streaming can also be aggregated without onChunk, but then the caller\n    // cannot consume incremental output, which is the purpose of this option.\n    // Environment-enabled streaming supports that aggregation-only use case.\n    // Once onChunk delivers a chunk, later errors are not retried to avoid\n    // delivering duplicate content. Errors before the first chunk can retry.\n    assert(\n      typeof options?.onChunk === 'function',\n      'onChunk is required when stream: true is explicitly specified',\n    );\n  }\n\n  // Low-level callers without a TaskRunner still need a stable ID for the\n  // lifetime of this model call (including its network retries).\n  const executionId = modelRuntime.executionId ?? `unscoped-${uuid()}`;\n\n  // Stable internal ID for this call, used by the agent to deduplicate usage\n  // across the onUsage callback and the task-dump-based collectUsageMetrics()\n  // path when the provider does not return a request_id.\n  const internalCallId = nextInternalCallId();\n\n  const { config: modelConfig } = modelRuntime;\n\n  const prepare = async (): Promise<PreparedModelInput> => {\n    const input = { messages, modelRuntime, options };\n    if (isCodexAppServerProvider(modelConfig.openaiBaseURL)) {\n      return { protocol: 'codex', input: prepareCodexCall(input) };\n    }\n    const protocol = modelConfig.protocol ?? DEFAULT_MODEL_PROTOCOL;\n\n    assert(\n      modelRuntime.adapter.supportedProtocols.includes(protocol),\n      `Model adapter \"${modelConfig.modelFamily ?? 'default'}\" does not support protocol \"${protocol}\"`,\n    );\n    if (protocol === 'openai-responses') {\n      return { protocol: 'responses', input: await prepareResponses(input) };\n    }\n    return {\n      protocol: 'chat-completion',\n      input: await prepareChatCompletion(input),\n    };\n  };\n\n  const prepared = options?.abortSignal\n    ? await runWithAbortSignal(options.abortSignal, prepare)\n    : await prepare();\n\n  const result = await callModelWithRetry({\n    messages,\n    modelRuntime,\n    options,\n    executionId,\n    internalCallId,\n    prepared,\n  });\n\n  const {\n    rawUsage,\n    timeCost,\n    totalTimeCost,\n    retryCount,\n    requestId,\n    responseModelName,\n    ...response\n  } = result;\n\n  const usage = buildUsageInfo({\n    apiProtocol: prepared.protocol,\n    stream: response.isStreamed,\n    usageData: rawUsage,\n    timeCost,\n    totalTimeCost,\n    retryCount,\n    requestId,\n    responseModelName,\n    modelName: modelConfig.modelName,\n    modelDescription: modelConfig.modelDescription,\n    slot: modelConfig.slot,\n    internalCallId,\n  });\n\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n  debugProfileStats(\n    `model, ${modelConfig.modelName}, mode, ${modelConfig.modelFamily || 'default'}, streaming, ${response.isStreamed}, prompt-tokens, ${usage?.prompt_tokens ?? ''}, completion-tokens, ${usage?.completion_tokens ?? ''}, total-tokens, ${usage?.total_tokens ?? ''}, cached-input, ${usage?.cached_input ?? ''}, cost-ms, ${timeCost ?? ''}, requestId, ${requestId ?? ''}, slot, ${modelConfig.slot}, configured-temperature, ${modelConfig.temperature ?? ''}`,\n  );\n  debugProfileDetail(`model usage detail: ${JSON.stringify(usage)}`);\n\n  if (usage && modelRuntime.onUsage) {\n    modelRuntime.onUsage(usage);\n  }\n  return { ...response, usage };\n}\n\ntype PreparedModelInput =\n  | { protocol: 'codex'; input: ReturnType<typeof prepareCodexCall> }\n  | PreparedOpenAIInput;\n\ntype ModelCallInput = {\n  messages: ModelCallMessages;\n  modelRuntime: ModelRuntime;\n  options?: CallAIOptions;\n  executionId: string;\n  internalCallId: string;\n  prepared: PreparedModelInput;\n};\n\n/** Carries the original failure across the retry boundary without replaying output. */\nclass NonRetryableModelCallError extends Error {\n  readonly cause: Error;\n\n  constructor(cause: Error) {\n    super(cause.message, { cause });\n    this.cause = cause;\n  }\n}\n\nasync function callModelWithRetry(\n  input: ModelCallInput,\n): Promise<ModelCallResult & { totalTimeCost: number; retryCount: number }> {\n  const startTime = Date.now();\n  const {\n    modelRuntime: { config },\n    options,\n  } = input;\n  const maxAttempts = normalizeRetryCount(config.retryCount) + 1;\n  const retryInterval = config.retryInterval ?? 2000;\n  const attemptErrors: Array<{ attempt: number; error: unknown }> = [];\n\n  for (let attempt = 1; ; attempt++) {\n    options?.abortSignal?.throwIfAborted();\n    try {\n      const result = await callModelOnce(input, attempt);\n      return {\n        ...result,\n        totalTimeCost: Date.now() - startTime,\n        retryCount: attempt - 1,\n      };\n    } catch (error) {\n      options?.abortSignal?.throwIfAborted();\n      if (error instanceof NonRetryableModelCallError) {\n        throw error.cause;\n      }\n      const lastError = toError(error);\n      attemptErrors.push({ attempt, error: lastError });\n      if (attempt === maxAttempts) {\n        throw appendAIRequestFailureSummary(\n          lastError,\n          attemptErrors,\n          maxAttempts,\n        );\n      }\n      getDebug('ai:call', { console: true })(\n        `AI call failed (attempt ${attempt}/${maxAttempts}), retrying in ${retryInterval}ms... Error: ${lastError.message}`,\n      );\n    }\n    await waitForRetry(retryInterval, options?.abortSignal);\n  }\n}\n\nasync function callModelOnce(\n  {\n    messages,\n    modelRuntime,\n    options,\n    executionId,\n    internalCallId,\n    prepared,\n  }: ModelCallInput,\n  attempt: number,\n): Promise<ModelCallResult> {\n  const { config: modelConfig } = modelRuntime;\n  const recordingEnabled = isModelCallRecordingEnabled();\n  const recordEvent =\n    recordingEnabled || modelRuntime.onRequest\n      ? (event: Record<string, unknown>) => {\n          if (event.type === 'request') {\n            const request = event.request as { url?: string } | undefined;\n            modelRuntime.onRequest?.(request?.url);\n          }\n          if (!recordingEnabled) return;\n          void recordModelCallEvent({\n            executionId,\n            callId: internalCallId,\n            semanticRetryAttempt: options?.semanticRetryAttempt,\n            slot: modelConfig.slot,\n            intent: modelConfig.intent,\n            modelFamily: modelConfig.modelFamily,\n            ...event,\n            attempt,\n          });\n        }\n      : undefined;\n  const effectiveTimeoutMs = resolveEffectiveTimeoutMs(modelConfig.timeout);\n  const { signal: requestSignal, cleanup } = buildRequestAbortSignal(\n    effectiveTimeoutMs,\n    options?.abortSignal,\n  );\n  let deliveredChunk = false;\n  const onChunk = options?.onChunk;\n  const context: ModelCallContext = {\n    messages,\n    modelRuntime,\n    executionId,\n    recordEvent,\n    requestSignal,\n    effectiveTimeoutMs,\n    options: onChunk\n      ? {\n          ...options,\n          onChunk: (chunk) => {\n            requestSignal.throwIfAborted();\n            deliveredChunk = true;\n            onChunk(chunk);\n          },\n        }\n      : options,\n  };\n  try {\n    return await runWithAbortSignal(requestSignal, () =>\n      prepared.protocol === 'codex'\n        ? callCodex(context, prepared.input)\n        : callOpenAI(context, prepared),\n    );\n  } catch (error) {\n    options?.abortSignal?.throwIfAborted();\n    const lastError = toError(error);\n    if (deliveredChunk) {\n      throw new NonRetryableModelCallError(lastError);\n    }\n    throw lastError;\n  } finally {\n    cleanup();\n  }\n}\n"],"names":["DEFAULT_MODEL_PROTOCOL","callAI","messages","modelRuntime","options","isStreaming","assert","executionId","uuid","internalCallId","nextInternalCallId","modelConfig","prepare","input","isCodexAppServerProvider","prepareCodexCall","protocol","prepareResponses","prepareChatCompletion","prepared","runWithAbortSignal","result","callModelWithRetry","rawUsage","timeCost","totalTimeCost","retryCount","requestId","responseModelName","response","usage","buildUsageInfo","debugProfileStats","getDebug","debugProfileDetail","JSON","NonRetryableModelCallError","Error","cause","startTime","Date","config","maxAttempts","normalizeRetryCount","retryInterval","attemptErrors","attempt","callModelOnce","error","lastError","toError","appendAIRequestFailureSummary","waitForRetry","recordingEnabled","isModelCallRecordingEnabled","recordEvent","event","request","recordModelCallEvent","undefined","effectiveTimeoutMs","resolveEffectiveTimeoutMs","requestSignal","cleanup","buildRequestAbortSignal","deliveredChunk","onChunk","context","chunk","callCodex","callOpenAI"],"mappings":";;;;;;;;;;;;;;;;;;;;AAiCA,MAAMA,yBAAyB;AAExB,eAAeC,OACpBC,QAA2B,EAC3BC,YAA0B,EAC1BC,OAAuB;IAEvBA,SAAS,aAAa;IACtB,MAAMC,cAAcD,SAAS,WAAW;IACxC,IAAIC,aAOFC,OACE,AAA4B,cAA5B,OAAOF,SAAS,SAChB;IAMJ,MAAMG,cAAcJ,aAAa,WAAW,IAAI,CAAC,SAAS,EAAEK,QAAQ;IAKpE,MAAMC,iBAAiBC;IAEvB,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGR;IAEhC,MAAMS,UAAU;QACd,MAAMC,QAAQ;YAAEX;YAAUC;YAAcC;QAAQ;QAChD,IAAIU,yBAAyBH,YAAY,aAAa,GACpD,OAAO;YAAE,UAAU;YAAS,OAAOI,iBAAiBF;QAAO;QAE7D,MAAMG,WAAWL,YAAY,QAAQ,IAAIX;QAEzCM,OACEH,aAAa,OAAO,CAAC,kBAAkB,CAAC,QAAQ,CAACa,WACjD,CAAC,eAAe,EAAEL,YAAY,WAAW,IAAI,UAAU,6BAA6B,EAAEK,SAAS,CAAC,CAAC;QAEnG,IAAIA,AAAa,uBAAbA,UACF,OAAO;YAAE,UAAU;YAAa,OAAO,MAAMC,iBAAiBJ;QAAO;QAEvE,OAAO;YACL,UAAU;YACV,OAAO,MAAMK,sBAAsBL;QACrC;IACF;IAEA,MAAMM,WAAWf,SAAS,cACtB,MAAMgB,mBAAmBhB,QAAQ,WAAW,EAAEQ,WAC9C,MAAMA;IAEV,MAAMS,SAAS,MAAMC,mBAAmB;QACtCpB;QACAC;QACAC;QACAG;QACAE;QACAU;IACF;IAEA,MAAM,EACJI,QAAQ,EACRC,QAAQ,EACRC,aAAa,EACbC,UAAU,EACVC,SAAS,EACTC,iBAAiB,EACjB,GAAGC,UACJ,GAAGR;IAEJ,MAAMS,QAAQC,eAAe;QAC3B,aAAaZ,SAAS,QAAQ;QAC9B,QAAQU,SAAS,UAAU;QAC3B,WAAWN;QACXC;QACAC;QACAC;QACAC;QACAC;QACA,WAAWjB,YAAY,SAAS;QAChC,kBAAkBA,YAAY,gBAAgB;QAC9C,MAAMA,YAAY,IAAI;QACtBF;IACF;IAEA,MAAMuB,oBAAoBC,SAAS;IACnC,MAAMC,qBAAqBD,SAAS;IACpCD,kBACE,CAAC,OAAO,EAAErB,YAAY,SAAS,CAAC,QAAQ,EAAEA,YAAY,WAAW,IAAI,UAAU,aAAa,EAAEkB,SAAS,UAAU,CAAC,iBAAiB,EAAEC,OAAO,iBAAiB,GAAG,qBAAqB,EAAEA,OAAO,qBAAqB,GAAG,gBAAgB,EAAEA,OAAO,gBAAgB,GAAG,gBAAgB,EAAEA,OAAO,gBAAgB,GAAG,WAAW,EAAEN,YAAY,GAAG,aAAa,EAAEG,aAAa,GAAG,QAAQ,EAAEhB,YAAY,IAAI,CAAC,0BAA0B,EAAEA,YAAY,WAAW,IAAI,IAAI;IAEjcuB,mBAAmB,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACL,QAAQ;IAEjE,IAAIA,SAAS3B,aAAa,OAAO,EAC/BA,aAAa,OAAO,CAAC2B;IAEvB,OAAO;QAAE,GAAGD,QAAQ;QAAEC;IAAM;AAC9B;AAgBA,MAAMM,mCAAmCC;IAGvC,YAAYC,KAAY,CAAE;QACxB,KAAK,CAACA,MAAM,OAAO,EAAE;YAAEA;QAAM,IAH/B,uBAAS,SAAT;QAIE,IAAI,CAAC,KAAK,GAAGA;IACf;AACF;AAEA,eAAehB,mBACbT,KAAqB;IAErB,MAAM0B,YAAYC,KAAK,GAAG;IAC1B,MAAM,EACJ,cAAc,EAAEC,MAAM,EAAE,EACxBrC,OAAO,EACR,GAAGS;IACJ,MAAM6B,cAAcC,oBAAoBF,OAAO,UAAU,IAAI;IAC7D,MAAMG,gBAAgBH,OAAO,aAAa,IAAI;IAC9C,MAAMI,gBAA4D,EAAE;IAEpE,IAAK,IAAIC,UAAU,IAAKA,UAAW;QACjC1C,SAAS,aAAa;QACtB,IAAI;YACF,MAAMiB,SAAS,MAAM0B,cAAclC,OAAOiC;YAC1C,OAAO;gBACL,GAAGzB,MAAM;gBACT,eAAemB,KAAK,GAAG,KAAKD;gBAC5B,YAAYO,UAAU;YACxB;QACF,EAAE,OAAOE,OAAO;YACd5C,SAAS,aAAa;YACtB,IAAI4C,iBAAiBZ,4BACnB,MAAMY,MAAM,KAAK;YAEnB,MAAMC,YAAYC,QAAQF;YAC1BH,cAAc,IAAI,CAAC;gBAAEC;gBAAS,OAAOG;YAAU;YAC/C,IAAIH,YAAYJ,aACd,MAAMS,8BACJF,WACAJ,eACAH;YAGJT,SAAS,WAAW;gBAAE,SAAS;YAAK,GAClC,CAAC,wBAAwB,EAAEa,QAAQ,CAAC,EAAEJ,YAAY,eAAe,EAAEE,cAAc,aAAa,EAAEK,UAAU,OAAO,EAAE;QAEvH;QACA,MAAMG,aAAaR,eAAexC,SAAS;IAC7C;AACF;AAEA,eAAe2C,cACb,EACE7C,QAAQ,EACRC,YAAY,EACZC,OAAO,EACPG,WAAW,EACXE,cAAc,EACdU,QAAQ,EACO,EACjB2B,OAAe;IAEf,MAAM,EAAE,QAAQnC,WAAW,EAAE,GAAGR;IAChC,MAAMkD,mBAAmBC;IACzB,MAAMC,cACJF,oBAAoBlD,aAAa,SAAS,GACtC,CAACqD;QACC,IAAIA,AAAe,cAAfA,MAAM,IAAI,EAAgB;YAC5B,MAAMC,UAAUD,MAAM,OAAO;YAC7BrD,aAAa,SAAS,GAAGsD,SAAS;QACpC;QACA,IAAI,CAACJ,kBAAkB;QAClBK,qBAAqB;YACxBnD;YACA,QAAQE;YACR,sBAAsBL,SAAS;YAC/B,MAAMO,YAAY,IAAI;YACtB,QAAQA,YAAY,MAAM;YAC1B,aAAaA,YAAY,WAAW;YACpC,GAAG6C,KAAK;YACRV;QACF;IACF,IACAa;IACN,MAAMC,qBAAqBC,0BAA0BlD,YAAY,OAAO;IACxE,MAAM,EAAE,QAAQmD,aAAa,EAAEC,OAAO,EAAE,GAAGC,wBACzCJ,oBACAxD,SAAS;IAEX,IAAI6D,iBAAiB;IACrB,MAAMC,UAAU9D,SAAS;IACzB,MAAM+D,UAA4B;QAChCjE;QACAC;QACAI;QACAgD;QACAO;QACAF;QACA,SAASM,UACL;YACE,GAAG9D,OAAO;YACV,SAAS,CAACgE;gBACRN,cAAc,cAAc;gBAC5BG,iBAAiB;gBACjBC,QAAQE;YACV;QACF,IACAhE;IACN;IACA,IAAI;QACF,OAAO,MAAMgB,mBAAmB0C,eAAe,IAC7C3C,AAAsB,YAAtBA,SAAS,QAAQ,GACbkD,UAAUF,SAAShD,SAAS,KAAK,IACjCmD,WAAWH,SAAShD;IAE5B,EAAE,OAAO6B,OAAO;QACd5C,SAAS,aAAa;QACtB,MAAM6C,YAAYC,QAAQF;QAC1B,IAAIiB,gBACF,MAAM,IAAI7B,2BAA2Ba;QAEvC,MAAMA;IACR,SAAU;QACRc;IACF;AACF"}
+{"version":3,"file":"ai-model/service-caller/call-ai.mjs","sources":["../../../../src/ai-model/service-caller/call-ai.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport { assert, uuid } from '@midscene/shared/utils';\nimport type { ModelRuntime } from '../models';\nimport { callCodex, prepareCodexCall } from './codex/call-codex';\nimport { isCodexAppServerProvider } from './codex/codex-app-server';\nimport {\n  isModelCallRecordingEnabled,\n  recordModelCallEvent,\n} from './model-call-recorder';\nimport { type PreparedOpenAIInput, callOpenAI } from './openai/call-openai';\nimport { prepareChatCompletion } from './openai/chat-completion/chat-completion';\nimport { prepareResponses } from './openai/responses/responses';\nimport {\n  buildRequestAbortSignal,\n  resolveEffectiveTimeoutMs,\n  runWithAbortSignal,\n  waitForRetry,\n} from './request-timeout';\nimport { callSystemOne, prepareSystemOne } from './system-one';\nimport type { ModelCallMessages } from './types';\nimport type {\n  AICallResult,\n  CallAIOptions,\n  ModelCallContext,\n  ModelCallResult,\n} from './types';\nimport {\n  appendAIRequestFailureSummary,\n  buildUsageInfo,\n  nextInternalCallId,\n  normalizeRetryCount,\n  toError,\n} from './utils';\n\nconst DEFAULT_MODEL_PROTOCOL = 'openai-chat';\n\nexport async function callAI(\n  messages: ModelCallMessages,\n  modelRuntime: ModelRuntime,\n  options?: CallAIOptions,\n): Promise<AICallResult> {\n  options?.abortSignal?.throwIfAborted();\n  const isStreaming = options?.stream === true;\n  if (isStreaming) {\n    // Explicit streaming is currently used only by recorder code generation.\n    // Streaming can also be aggregated without onChunk, but then the caller\n    // cannot consume incremental output, which is the purpose of this option.\n    // Environment-enabled streaming supports that aggregation-only use case.\n    // Once onChunk delivers a chunk, later errors are not retried to avoid\n    // delivering duplicate content. Errors before the first chunk can retry.\n    assert(\n      typeof options?.onChunk === 'function',\n      'onChunk is required when stream: true is explicitly specified',\n    );\n  }\n\n  // Low-level callers without a TaskRunner still need a stable ID for the\n  // lifetime of this model call (including its network retries).\n  const executionId = modelRuntime.executionId ?? `unscoped-${uuid()}`;\n\n  // Stable internal ID for this call, used by the agent to deduplicate usage\n  // across the onUsage callback and the task-dump-based collectUsageMetrics()\n  // path when the provider does not return a request_id.\n  const internalCallId = nextInternalCallId();\n\n  const { config: modelConfig } = modelRuntime;\n\n  const prepare = async (): Promise<PreparedModelInput> => {\n    if (options?.systemOneRequest) {\n      assert(\n        modelConfig.modelFamily === 'jev',\n        'System One requests require the Jev model family',\n      );\n      return {\n        protocol: 'system-one',\n        input: await prepareSystemOne(modelConfig, options.systemOneRequest),\n      };\n    }\n    const input = { messages, modelRuntime, options };\n    if (isCodexAppServerProvider(modelConfig.openaiBaseURL)) {\n      return { protocol: 'codex', input: prepareCodexCall(input) };\n    }\n    const protocol = modelConfig.protocol ?? DEFAULT_MODEL_PROTOCOL;\n\n    assert(\n      modelRuntime.adapter.supportedProtocols.includes(protocol),\n      `Model adapter \"${modelConfig.modelFamily ?? 'default'}\" does not support protocol \"${protocol}\"`,\n    );\n    if (protocol === 'openai-responses') {\n      return { protocol: 'responses', input: await prepareResponses(input) };\n    }\n    return {\n      protocol: 'chat-completion',\n      input: await prepareChatCompletion(input),\n    };\n  };\n\n  const prepared = options?.abortSignal\n    ? await runWithAbortSignal(options.abortSignal, prepare)\n    : await prepare();\n\n  const result = await callModelWithRetry({\n    messages,\n    modelRuntime,\n    options,\n    executionId,\n    internalCallId,\n    prepared,\n  });\n\n  const {\n    rawUsage,\n    timeCost,\n    totalTimeCost,\n    retryCount,\n    requestId,\n    responseModelName,\n    ...response\n  } = result;\n\n  const rawUsageInfo = buildUsageInfo({\n    apiProtocol: prepared.protocol,\n    stream: response.isStreamed,\n    usageData: rawUsage,\n    timeCost,\n    totalTimeCost,\n    retryCount,\n    requestId,\n    responseModelName,\n    modelName: modelConfig.modelName,\n    modelDescription: modelConfig.modelDescription,\n    slot: modelConfig.slot,\n    internalCallId,\n  });\n\n  const usage = rawUsageInfo\n    ? { ...rawUsageInfo, modelRouting: modelRuntime.modelRouting }\n    : undefined;\n\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n  debugProfileStats(\n    `model, ${modelConfig.modelName}, mode, ${modelConfig.modelFamily || 'default'}, streaming, ${response.isStreamed}, prompt-tokens, ${usage?.prompt_tokens ?? ''}, completion-tokens, ${usage?.completion_tokens ?? ''}, total-tokens, ${usage?.total_tokens ?? ''}, cached-input, ${usage?.cached_input ?? ''}, cost-ms, ${timeCost ?? ''}, requestId, ${requestId ?? ''}, slot, ${modelConfig.slot}, configured-temperature, ${modelConfig.temperature ?? ''}`,\n  );\n  debugProfileDetail(`model usage detail: ${JSON.stringify(usage)}`);\n\n  if (usage && modelRuntime.onUsage) {\n    modelRuntime.onUsage(usage);\n  }\n  return { ...response, usage };\n}\n\ntype PreparedModelInput =\n  | {\n      protocol: 'system-one';\n      input: Awaited<ReturnType<typeof prepareSystemOne>>;\n    }\n  | { protocol: 'codex'; input: ReturnType<typeof prepareCodexCall> }\n  | PreparedOpenAIInput;\n\ntype ModelCallInput = {\n  messages: ModelCallMessages;\n  modelRuntime: ModelRuntime;\n  options?: CallAIOptions;\n  executionId: string;\n  internalCallId: string;\n  prepared: PreparedModelInput;\n};\n\n/** Carries the original failure across the retry boundary without replaying output. */\nclass NonRetryableModelCallError extends Error {\n  readonly cause: Error;\n\n  constructor(cause: Error) {\n    super(cause.message, { cause });\n    this.cause = cause;\n  }\n}\n\nasync function callModelWithRetry(\n  input: ModelCallInput,\n): Promise<ModelCallResult & { totalTimeCost: number; retryCount: number }> {\n  const startTime = Date.now();\n  const {\n    modelRuntime: { config },\n    options,\n  } = input;\n  const maxAttempts = normalizeRetryCount(config.retryCount) + 1;\n  const retryInterval = config.retryInterval ?? 2000;\n  const attemptErrors: Array<{ attempt: number; error: unknown }> = [];\n\n  for (let attempt = 1; ; attempt++) {\n    options?.abortSignal?.throwIfAborted();\n    try {\n      const result = await callModelOnce(input, attempt);\n      return {\n        ...result,\n        totalTimeCost: Date.now() - startTime,\n        retryCount: attempt - 1,\n      };\n    } catch (error) {\n      options?.abortSignal?.throwIfAborted();\n      if (error instanceof NonRetryableModelCallError) {\n        throw error.cause;\n      }\n      const lastError = toError(error);\n      attemptErrors.push({ attempt, error: lastError });\n      if (attempt === maxAttempts) {\n        throw appendAIRequestFailureSummary(\n          lastError,\n          attemptErrors,\n          maxAttempts,\n        );\n      }\n      getDebug('ai:call', { console: true })(\n        `AI call failed (attempt ${attempt}/${maxAttempts}), retrying in ${retryInterval}ms... Error: ${lastError.message}`,\n      );\n    }\n    await waitForRetry(retryInterval, options?.abortSignal);\n  }\n}\n\nasync function callModelOnce(\n  {\n    messages,\n    modelRuntime,\n    options,\n    executionId,\n    internalCallId,\n    prepared,\n  }: ModelCallInput,\n  attempt: number,\n): Promise<ModelCallResult> {\n  const { config: modelConfig } = modelRuntime;\n  const recordingEnabled = isModelCallRecordingEnabled();\n  const recordEvent =\n    recordingEnabled || modelRuntime.onRequest\n      ? (event: Record<string, unknown>) => {\n          if (event.type === 'request') {\n            const request = event.request as { url?: string } | undefined;\n            modelRuntime.onRequest?.(request?.url);\n          }\n          if (!recordingEnabled) return;\n          void recordModelCallEvent({\n            executionId,\n            callId: internalCallId,\n            semanticRetryAttempt: options?.semanticRetryAttempt,\n            slot: modelConfig.slot,\n            intent: modelConfig.intent,\n            modelFamily: modelConfig.modelFamily,\n            modelRouting: modelRuntime.modelRouting,\n            ...event,\n            attempt,\n          });\n        }\n      : undefined;\n  const effectiveTimeoutMs = resolveEffectiveTimeoutMs(modelConfig.timeout);\n  const { signal: requestSignal, cleanup } = buildRequestAbortSignal(\n    effectiveTimeoutMs,\n    options?.abortSignal,\n  );\n  let deliveredChunk = false;\n  const onChunk = options?.onChunk;\n  const context: ModelCallContext = {\n    messages,\n    modelRuntime,\n    executionId,\n    recordEvent,\n    requestSignal,\n    effectiveTimeoutMs,\n    options: onChunk\n      ? {\n          ...options,\n          onChunk: (chunk) => {\n            requestSignal.throwIfAborted();\n            deliveredChunk = true;\n            onChunk(chunk);\n          },\n        }\n      : options,\n  };\n  try {\n    return await runWithAbortSignal(requestSignal, () =>\n      prepared.protocol === 'system-one'\n        ? callSystemOne(context, prepared.input)\n        : prepared.protocol === 'codex'\n          ? callCodex(context, prepared.input)\n          : callOpenAI(context, prepared),\n    );\n  } catch (error) {\n    options?.abortSignal?.throwIfAborted();\n    const lastError = toError(error);\n    if (deliveredChunk) {\n      throw new NonRetryableModelCallError(lastError);\n    }\n    throw lastError;\n  } finally {\n    cleanup();\n  }\n}\n"],"names":["DEFAULT_MODEL_PROTOCOL","callAI","messages","modelRuntime","options","isStreaming","assert","executionId","uuid","internalCallId","nextInternalCallId","modelConfig","prepare","prepareSystemOne","input","isCodexAppServerProvider","prepareCodexCall","protocol","prepareResponses","prepareChatCompletion","prepared","runWithAbortSignal","result","callModelWithRetry","rawUsage","timeCost","totalTimeCost","retryCount","requestId","responseModelName","response","rawUsageInfo","buildUsageInfo","usage","undefined","debugProfileStats","getDebug","debugProfileDetail","JSON","NonRetryableModelCallError","Error","cause","startTime","Date","config","maxAttempts","normalizeRetryCount","retryInterval","attemptErrors","attempt","callModelOnce","error","lastError","toError","appendAIRequestFailureSummary","waitForRetry","recordingEnabled","isModelCallRecordingEnabled","recordEvent","event","request","recordModelCallEvent","effectiveTimeoutMs","resolveEffectiveTimeoutMs","requestSignal","cleanup","buildRequestAbortSignal","deliveredChunk","onChunk","context","chunk","callSystemOne","callCodex","callOpenAI"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAkCA,MAAMA,yBAAyB;AAExB,eAAeC,OACpBC,QAA2B,EAC3BC,YAA0B,EAC1BC,OAAuB;IAEvBA,SAAS,aAAa;IACtB,MAAMC,cAAcD,SAAS,WAAW;IACxC,IAAIC,aAOFC,OACE,AAA4B,cAA5B,OAAOF,SAAS,SAChB;IAMJ,MAAMG,cAAcJ,aAAa,WAAW,IAAI,CAAC,SAAS,EAAEK,QAAQ;IAKpE,MAAMC,iBAAiBC;IAEvB,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGR;IAEhC,MAAMS,UAAU;QACd,IAAIR,SAAS,kBAAkB;YAC7BE,OACEK,AAA4B,UAA5BA,YAAY,WAAW,EACvB;YAEF,OAAO;gBACL,UAAU;gBACV,OAAO,MAAME,iBAAiBF,aAAaP,QAAQ,gBAAgB;YACrE;QACF;QACA,MAAMU,QAAQ;YAAEZ;YAAUC;YAAcC;QAAQ;QAChD,IAAIW,yBAAyBJ,YAAY,aAAa,GACpD,OAAO;YAAE,UAAU;YAAS,OAAOK,iBAAiBF;QAAO;QAE7D,MAAMG,WAAWN,YAAY,QAAQ,IAAIX;QAEzCM,OACEH,aAAa,OAAO,CAAC,kBAAkB,CAAC,QAAQ,CAACc,WACjD,CAAC,eAAe,EAAEN,YAAY,WAAW,IAAI,UAAU,6BAA6B,EAAEM,SAAS,CAAC,CAAC;QAEnG,IAAIA,AAAa,uBAAbA,UACF,OAAO;YAAE,UAAU;YAAa,OAAO,MAAMC,iBAAiBJ;QAAO;QAEvE,OAAO;YACL,UAAU;YACV,OAAO,MAAMK,sBAAsBL;QACrC;IACF;IAEA,MAAMM,WAAWhB,SAAS,cACtB,MAAMiB,mBAAmBjB,QAAQ,WAAW,EAAEQ,WAC9C,MAAMA;IAEV,MAAMU,SAAS,MAAMC,mBAAmB;QACtCrB;QACAC;QACAC;QACAG;QACAE;QACAW;IACF;IAEA,MAAM,EACJI,QAAQ,EACRC,QAAQ,EACRC,aAAa,EACbC,UAAU,EACVC,SAAS,EACTC,iBAAiB,EACjB,GAAGC,UACJ,GAAGR;IAEJ,MAAMS,eAAeC,eAAe;QAClC,aAAaZ,SAAS,QAAQ;QAC9B,QAAQU,SAAS,UAAU;QAC3B,WAAWN;QACXC;QACAC;QACAC;QACAC;QACAC;QACA,WAAWlB,YAAY,SAAS;QAChC,kBAAkBA,YAAY,gBAAgB;QAC9C,MAAMA,YAAY,IAAI;QACtBF;IACF;IAEA,MAAMwB,QAAQF,eACV;QAAE,GAAGA,YAAY;QAAE,cAAc5B,aAAa,YAAY;IAAC,IAC3D+B;IAEJ,MAAMC,oBAAoBC,SAAS;IACnC,MAAMC,qBAAqBD,SAAS;IACpCD,kBACE,CAAC,OAAO,EAAExB,YAAY,SAAS,CAAC,QAAQ,EAAEA,YAAY,WAAW,IAAI,UAAU,aAAa,EAAEmB,SAAS,UAAU,CAAC,iBAAiB,EAAEG,OAAO,iBAAiB,GAAG,qBAAqB,EAAEA,OAAO,qBAAqB,GAAG,gBAAgB,EAAEA,OAAO,gBAAgB,GAAG,gBAAgB,EAAEA,OAAO,gBAAgB,GAAG,WAAW,EAAER,YAAY,GAAG,aAAa,EAAEG,aAAa,GAAG,QAAQ,EAAEjB,YAAY,IAAI,CAAC,0BAA0B,EAAEA,YAAY,WAAW,IAAI,IAAI;IAEjc0B,mBAAmB,CAAC,oBAAoB,EAAEC,KAAK,SAAS,CAACL,QAAQ;IAEjE,IAAIA,SAAS9B,aAAa,OAAO,EAC/BA,aAAa,OAAO,CAAC8B;IAEvB,OAAO;QAAE,GAAGH,QAAQ;QAAEG;IAAM;AAC9B;AAoBA,MAAMM,mCAAmCC;IAGvC,YAAYC,KAAY,CAAE;QACxB,KAAK,CAACA,MAAM,OAAO,EAAE;YAAEA;QAAM,IAH/B,uBAAS,SAAT;QAIE,IAAI,CAAC,KAAK,GAAGA;IACf;AACF;AAEA,eAAelB,mBACbT,KAAqB;IAErB,MAAM4B,YAAYC,KAAK,GAAG;IAC1B,MAAM,EACJ,cAAc,EAAEC,MAAM,EAAE,EACxBxC,OAAO,EACR,GAAGU;IACJ,MAAM+B,cAAcC,oBAAoBF,OAAO,UAAU,IAAI;IAC7D,MAAMG,gBAAgBH,OAAO,aAAa,IAAI;IAC9C,MAAMI,gBAA4D,EAAE;IAEpE,IAAK,IAAIC,UAAU,IAAKA,UAAW;QACjC7C,SAAS,aAAa;QACtB,IAAI;YACF,MAAMkB,SAAS,MAAM4B,cAAcpC,OAAOmC;YAC1C,OAAO;gBACL,GAAG3B,MAAM;gBACT,eAAeqB,KAAK,GAAG,KAAKD;gBAC5B,YAAYO,UAAU;YACxB;QACF,EAAE,OAAOE,OAAO;YACd/C,SAAS,aAAa;YACtB,IAAI+C,iBAAiBZ,4BACnB,MAAMY,MAAM,KAAK;YAEnB,MAAMC,YAAYC,QAAQF;YAC1BH,cAAc,IAAI,CAAC;gBAAEC;gBAAS,OAAOG;YAAU;YAC/C,IAAIH,YAAYJ,aACd,MAAMS,8BACJF,WACAJ,eACAH;YAGJT,SAAS,WAAW;gBAAE,SAAS;YAAK,GAClC,CAAC,wBAAwB,EAAEa,QAAQ,CAAC,EAAEJ,YAAY,eAAe,EAAEE,cAAc,aAAa,EAAEK,UAAU,OAAO,EAAE;QAEvH;QACA,MAAMG,aAAaR,eAAe3C,SAAS;IAC7C;AACF;AAEA,eAAe8C,cACb,EACEhD,QAAQ,EACRC,YAAY,EACZC,OAAO,EACPG,WAAW,EACXE,cAAc,EACdW,QAAQ,EACO,EACjB6B,OAAe;IAEf,MAAM,EAAE,QAAQtC,WAAW,EAAE,GAAGR;IAChC,MAAMqD,mBAAmBC;IACzB,MAAMC,cACJF,oBAAoBrD,aAAa,SAAS,GACtC,CAACwD;QACC,IAAIA,AAAe,cAAfA,MAAM,IAAI,EAAgB;YAC5B,MAAMC,UAAUD,MAAM,OAAO;YAC7BxD,aAAa,SAAS,GAAGyD,SAAS;QACpC;QACA,IAAI,CAACJ,kBAAkB;QAClBK,qBAAqB;YACxBtD;YACA,QAAQE;YACR,sBAAsBL,SAAS;YAC/B,MAAMO,YAAY,IAAI;YACtB,QAAQA,YAAY,MAAM;YAC1B,aAAaA,YAAY,WAAW;YACpC,cAAcR,aAAa,YAAY;YACvC,GAAGwD,KAAK;YACRV;QACF;IACF,IACAf;IACN,MAAM4B,qBAAqBC,0BAA0BpD,YAAY,OAAO;IACxE,MAAM,EAAE,QAAQqD,aAAa,EAAEC,OAAO,EAAE,GAAGC,wBACzCJ,oBACA1D,SAAS;IAEX,IAAI+D,iBAAiB;IACrB,MAAMC,UAAUhE,SAAS;IACzB,MAAMiE,UAA4B;QAChCnE;QACAC;QACAI;QACAmD;QACAM;QACAF;QACA,SAASM,UACL;YACE,GAAGhE,OAAO;YACV,SAAS,CAACkE;gBACRN,cAAc,cAAc;gBAC5BG,iBAAiB;gBACjBC,QAAQE;YACV;QACF,IACAlE;IACN;IACA,IAAI;QACF,OAAO,MAAMiB,mBAAmB2C,eAAe,IAC7C5C,AAAsB,iBAAtBA,SAAS,QAAQ,GACbmD,cAAcF,SAASjD,SAAS,KAAK,IACrCA,AAAsB,YAAtBA,SAAS,QAAQ,GACfoD,UAAUH,SAASjD,SAAS,KAAK,IACjCqD,WAAWJ,SAASjD;IAE9B,EAAE,OAAO+B,OAAO;QACd/C,SAAS,aAAa;QACtB,MAAMgD,YAAYC,QAAQF;QAC1B,IAAIgB,gBACF,MAAM,IAAI5B,2BAA2Ba;QAEvC,MAAMA;IACR,SAAU;QACRa;IACF;AACF"}

package/dist/es/ai-model/service-caller/openai/call-openai.mjs

@@ -1,4 +1,3 @@
-import { MIDSCENE_MODEL_STREAM_MODE, globalConfigManager } from "@midscene/shared/env";
 import { getDebug } from "@midscene/shared/logger";
 import { AIResponseParseError } from "../utils.mjs";
 import { callChatCompletion } from "./chat-completion/chat-completion.mjs";
@@ -8,7 +7,7 @@ import { callResponses } from "./responses/responses.mjs";
 async function callOpenAI({ modelRuntime, options, executionId, recordEvent, requestSignal, effectiveTimeoutMs }, prepared) {
     const { config: modelConfig } = modelRuntime;
     const { modelName } = modelConfig;
-    const isStreaming = options?.stream ?? 'stream' === globalConfigManager.getEnvConfigValue(MIDSCENE_MODEL_STREAM_MODE);
+    const isStreaming = options?.stream ?? 'stream' === modelConfig.streamMode;
     const { openai, openAIRequestContext } = await createClient({
         modelConfig,
         proxyAgent: prepared.input.proxyAgent,

package/dist/es/ai-model/service-caller/openai/call-openai.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/openai/call-openai.mjs","sources":["../../../../../src/ai-model/service-caller/openai/call-openai.ts"],"sourcesContent":["import {\n  MIDSCENE_MODEL_STREAM_MODE,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ModelCallContext, ModelCallResult } from '../types';\nimport { AIResponseParseError } from '../utils';\nimport {\n  callChatCompletion,\n  type prepareChatCompletion,\n} from './chat-completion/chat-completion';\nimport { createClient } from './openai-client';\nimport { formatOpenAIAPIErrorDetails } from './openai-request-context';\nimport { callResponses, type prepareResponses } from './responses/responses';\n\nexport type PreparedOpenAIInput =\n  | {\n      protocol: 'chat-completion';\n      input: Awaited<ReturnType<typeof prepareChatCompletion>>;\n    }\n  | {\n      protocol: 'responses';\n      input: Awaited<ReturnType<typeof prepareResponses>>;\n    };\n\nexport async function callOpenAI(\n  {\n    modelRuntime,\n    options,\n    executionId,\n    recordEvent,\n    requestSignal,\n    effectiveTimeoutMs,\n  }: ModelCallContext,\n  prepared: PreparedOpenAIInput,\n): Promise<ModelCallResult> {\n  const { config: modelConfig } = modelRuntime;\n  const { modelName } = modelConfig;\n  const isStreaming =\n    options?.stream ??\n    globalConfigManager.getEnvConfigValue(MIDSCENE_MODEL_STREAM_MODE) ===\n      'stream';\n  const { openai, openAIRequestContext } = await createClient({\n    modelConfig,\n    proxyAgent: prepared.input.proxyAgent,\n    effectiveTimeoutMs,\n    executionId,\n    recordEvent,\n  });\n  const debugCall = getDebug('ai:call');\n  debugCall(\n    `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n  );\n  const requestOptions = {\n    requestSignal,\n    openAIRequestContext,\n    onChunk: options?.onChunk ?? (() => undefined),\n    recordEvent,\n  };\n  try {\n    const startTime = Date.now();\n    const response =\n      prepared.protocol === 'responses'\n        ? await callResponses(\n            {\n              ...requestOptions,\n              responses: openai.responses,\n              requestBodyParams: prepared.input.requestParams,\n            },\n            isStreaming,\n          )\n        : await callChatCompletion(\n            {\n              ...requestOptions,\n              completion: openai.chat.completions,\n              requestBodyParams: prepared.input.requestParams,\n              extractContentAndReasoning:\n                prepared.input.extractContentAndReasoning,\n              useReasoningAsContentFallback:\n                prepared.input.useReasoningAsContentFallback,\n            },\n            isStreaming,\n          );\n    const { reasoningContent, ...result } = response;\n    requestSignal.throwIfAborted();\n    const timeCost = Date.now() - startTime;\n    debugCall(`response reasoning content: ${reasoningContent}`);\n    debugCall(`response content: ${result.content}`);\n    recordEvent?.({\n      type: 'response',\n      http: openAIRequestContext.httpResponse,\n      final: {\n        content: result.content,\n        reasoningContent,\n        usage: result.rawUsage,\n        requestId: result.requestId,\n        timeCost,\n        responseModelName: result.responseModelName,\n      },\n    });\n    return {\n      ...result,\n      reasoning_content: reasoningContent,\n      isStreamed: isStreaming,\n      timeCost,\n    };\n  } catch (error: any) {\n    // Fetch already records non-2xx responses. Preserve successful HTTP\n    // responses too when SDK parsing or completion validation fails.\n    if (openAIRequestContext.httpResponse?.ok) {\n      recordEvent?.({\n        type: 'error',\n        ...openAIRequestContext.httpResponse,\n        error: error instanceof Error ? error.message : String(error),\n      });\n    }\n    getDebug('ai:call', { console: true })('call AI error', error);\n    if (error instanceof AIResponseParseError) {\n      throw error;\n    }\n    throw new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service (${modelName}): ${error.message}${formatOpenAIAPIErrorDetails(error, openAIRequestContext)}\\nTrouble shooting: https://midscenejs.com/model-provider.html`,\n      { cause: error },\n    );\n  }\n}\n"],"names":["callOpenAI","modelRuntime","options","executionId","recordEvent","requestSignal","effectiveTimeoutMs","prepared","modelConfig","modelName","isStreaming","globalConfigManager","MIDSCENE_MODEL_STREAM_MODE","openai","openAIRequestContext","createClient","debugCall","getDebug","requestOptions","undefined","startTime","Date","response","callResponses","callChatCompletion","reasoningContent","result","timeCost","error","Error","String","AIResponseParseError","formatOpenAIAPIErrorDetails"],"mappings":";;;;;;;AAyBO,eAAeA,WACpB,EACEC,YAAY,EACZC,OAAO,EACPC,WAAW,EACXC,WAAW,EACXC,aAAa,EACbC,kBAAkB,EACD,EACnBC,QAA6B;IAE7B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGP;IAChC,MAAM,EAAEQ,SAAS,EAAE,GAAGD;IACtB,MAAME,cACJR,SAAS,UACTS,AACE,aADFA,oBAAoB,iBAAiB,CAACC;IAExC,MAAM,EAAEC,MAAM,EAAEC,oBAAoB,EAAE,GAAG,MAAMC,aAAa;QAC1DP;QACA,YAAYD,SAAS,KAAK,CAAC,UAAU;QACrCD;QACAH;QACAC;IACF;IACA,MAAMY,YAAYC,SAAS;IAC3BD,UACE,CAAC,QAAQ,EAAEN,cAAc,eAAe,GAAG,WAAW,EAAED,WAAW;IAErE,MAAMS,iBAAiB;QACrBb;QACAS;QACA,SAASZ,SAAS,WAAa,KAAKiB,MAAQ;QAC5Cf;IACF;IACA,IAAI;QACF,MAAMgB,YAAYC,KAAK,GAAG;QAC1B,MAAMC,WACJf,AAAsB,gBAAtBA,SAAS,QAAQ,GACb,MAAMgB,cACJ;YACE,GAAGL,cAAc;YACjB,WAAWL,OAAO,SAAS;YAC3B,mBAAmBN,SAAS,KAAK,CAAC,aAAa;QACjD,GACAG,eAEF,MAAMc,mBACJ;YACE,GAAGN,cAAc;YACjB,YAAYL,OAAO,IAAI,CAAC,WAAW;YACnC,mBAAmBN,SAAS,KAAK,CAAC,aAAa;YAC/C,4BACEA,SAAS,KAAK,CAAC,0BAA0B;YAC3C,+BACEA,SAAS,KAAK,CAAC,6BAA6B;QAChD,GACAG;QAER,MAAM,EAAEe,gBAAgB,EAAE,GAAGC,QAAQ,GAAGJ;QACxCjB,cAAc,cAAc;QAC5B,MAAMsB,WAAWN,KAAK,GAAG,KAAKD;QAC9BJ,UAAU,CAAC,4BAA4B,EAAES,kBAAkB;QAC3DT,UAAU,CAAC,kBAAkB,EAAEU,OAAO,OAAO,EAAE;QAC/CtB,cAAc;YACZ,MAAM;YACN,MAAMU,qBAAqB,YAAY;YACvC,OAAO;gBACL,SAASY,OAAO,OAAO;gBACvBD;gBACA,OAAOC,OAAO,QAAQ;gBACtB,WAAWA,OAAO,SAAS;gBAC3BC;gBACA,mBAAmBD,OAAO,iBAAiB;YAC7C;QACF;QACA,OAAO;YACL,GAAGA,MAAM;YACT,mBAAmBD;YACnB,YAAYf;YACZiB;QACF;IACF,EAAE,OAAOC,OAAY;QAGnB,IAAId,qBAAqB,YAAY,EAAE,IACrCV,cAAc;YACZ,MAAM;YACN,GAAGU,qBAAqB,YAAY;YACpC,OAAOc,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QACzD;QAEFX,SAAS,WAAW;YAAE,SAAS;QAAK,GAAG,iBAAiBW;QACxD,IAAIA,iBAAiBG,sBACnB,MAAMH;QAER,MAAM,IAAIC,MACR,CAAC,eAAe,EAAEnB,cAAc,eAAe,GAAG,kBAAkB,EAAED,UAAU,GAAG,EAAEmB,MAAM,OAAO,GAAGI,4BAA4BJ,OAAOd,sBAAsB,8DAA8D,CAAC,EAC7N;YAAE,OAAOc;QAAM;IAEnB;AACF"}
+{"version":3,"file":"ai-model/service-caller/openai/call-openai.mjs","sources":["../../../../../src/ai-model/service-caller/openai/call-openai.ts"],"sourcesContent":["import { getDebug } from '@midscene/shared/logger';\nimport type { ModelCallContext, ModelCallResult } from '../types';\nimport { AIResponseParseError } from '../utils';\nimport {\n  callChatCompletion,\n  type prepareChatCompletion,\n} from './chat-completion/chat-completion';\nimport { createClient } from './openai-client';\nimport { formatOpenAIAPIErrorDetails } from './openai-request-context';\nimport { callResponses, type prepareResponses } from './responses/responses';\n\nexport type PreparedOpenAIInput =\n  | {\n      protocol: 'chat-completion';\n      input: Awaited<ReturnType<typeof prepareChatCompletion>>;\n    }\n  | {\n      protocol: 'responses';\n      input: Awaited<ReturnType<typeof prepareResponses>>;\n    };\n\nexport async function callOpenAI(\n  {\n    modelRuntime,\n    options,\n    executionId,\n    recordEvent,\n    requestSignal,\n    effectiveTimeoutMs,\n  }: ModelCallContext,\n  prepared: PreparedOpenAIInput,\n): Promise<ModelCallResult> {\n  const { config: modelConfig } = modelRuntime;\n  const { modelName } = modelConfig;\n  const isStreaming = options?.stream ?? modelConfig.streamMode === 'stream';\n  const { openai, openAIRequestContext } = await createClient({\n    modelConfig,\n    proxyAgent: prepared.input.proxyAgent,\n    effectiveTimeoutMs,\n    executionId,\n    recordEvent,\n  });\n  const debugCall = getDebug('ai:call');\n  debugCall(\n    `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n  );\n  const requestOptions = {\n    requestSignal,\n    openAIRequestContext,\n    onChunk: options?.onChunk ?? (() => undefined),\n    recordEvent,\n  };\n  try {\n    const startTime = Date.now();\n    const response =\n      prepared.protocol === 'responses'\n        ? await callResponses(\n            {\n              ...requestOptions,\n              responses: openai.responses,\n              requestBodyParams: prepared.input.requestParams,\n            },\n            isStreaming,\n          )\n        : await callChatCompletion(\n            {\n              ...requestOptions,\n              completion: openai.chat.completions,\n              requestBodyParams: prepared.input.requestParams,\n              extractContentAndReasoning:\n                prepared.input.extractContentAndReasoning,\n              useReasoningAsContentFallback:\n                prepared.input.useReasoningAsContentFallback,\n            },\n            isStreaming,\n          );\n    const { reasoningContent, ...result } = response;\n    requestSignal.throwIfAborted();\n    const timeCost = Date.now() - startTime;\n    debugCall(`response reasoning content: ${reasoningContent}`);\n    debugCall(`response content: ${result.content}`);\n    recordEvent?.({\n      type: 'response',\n      http: openAIRequestContext.httpResponse,\n      final: {\n        content: result.content,\n        reasoningContent,\n        usage: result.rawUsage,\n        requestId: result.requestId,\n        timeCost,\n        responseModelName: result.responseModelName,\n      },\n    });\n    return {\n      ...result,\n      reasoning_content: reasoningContent,\n      isStreamed: isStreaming,\n      timeCost,\n    };\n  } catch (error: any) {\n    // Fetch already records non-2xx responses. Preserve successful HTTP\n    // responses too when SDK parsing or completion validation fails.\n    if (openAIRequestContext.httpResponse?.ok) {\n      recordEvent?.({\n        type: 'error',\n        ...openAIRequestContext.httpResponse,\n        error: error instanceof Error ? error.message : String(error),\n      });\n    }\n    getDebug('ai:call', { console: true })('call AI error', error);\n    if (error instanceof AIResponseParseError) {\n      throw error;\n    }\n    throw new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service (${modelName}): ${error.message}${formatOpenAIAPIErrorDetails(error, openAIRequestContext)}\\nTrouble shooting: https://midscenejs.com/model-provider.html`,\n      { cause: error },\n    );\n  }\n}\n"],"names":["callOpenAI","modelRuntime","options","executionId","recordEvent","requestSignal","effectiveTimeoutMs","prepared","modelConfig","modelName","isStreaming","openai","openAIRequestContext","createClient","debugCall","getDebug","requestOptions","undefined","startTime","Date","response","callResponses","callChatCompletion","reasoningContent","result","timeCost","error","Error","String","AIResponseParseError","formatOpenAIAPIErrorDetails"],"mappings":";;;;;;AAqBO,eAAeA,WACpB,EACEC,YAAY,EACZC,OAAO,EACPC,WAAW,EACXC,WAAW,EACXC,aAAa,EACbC,kBAAkB,EACD,EACnBC,QAA6B;IAE7B,MAAM,EAAE,QAAQC,WAAW,EAAE,GAAGP;IAChC,MAAM,EAAEQ,SAAS,EAAE,GAAGD;IACtB,MAAME,cAAcR,SAAS,UAAUM,AAA2B,aAA3BA,YAAY,UAAU;IAC7D,MAAM,EAAEG,MAAM,EAAEC,oBAAoB,EAAE,GAAG,MAAMC,aAAa;QAC1DL;QACA,YAAYD,SAAS,KAAK,CAAC,UAAU;QACrCD;QACAH;QACAC;IACF;IACA,MAAMU,YAAYC,SAAS;IAC3BD,UACE,CAAC,QAAQ,EAAEJ,cAAc,eAAe,GAAG,WAAW,EAAED,WAAW;IAErE,MAAMO,iBAAiB;QACrBX;QACAO;QACA,SAASV,SAAS,WAAa,KAAKe,MAAQ;QAC5Cb;IACF;IACA,IAAI;QACF,MAAMc,YAAYC,KAAK,GAAG;QAC1B,MAAMC,WACJb,AAAsB,gBAAtBA,SAAS,QAAQ,GACb,MAAMc,cACJ;YACE,GAAGL,cAAc;YACjB,WAAWL,OAAO,SAAS;YAC3B,mBAAmBJ,SAAS,KAAK,CAAC,aAAa;QACjD,GACAG,eAEF,MAAMY,mBACJ;YACE,GAAGN,cAAc;YACjB,YAAYL,OAAO,IAAI,CAAC,WAAW;YACnC,mBAAmBJ,SAAS,KAAK,CAAC,aAAa;YAC/C,4BACEA,SAAS,KAAK,CAAC,0BAA0B;YAC3C,+BACEA,SAAS,KAAK,CAAC,6BAA6B;QAChD,GACAG;QAER,MAAM,EAAEa,gBAAgB,EAAE,GAAGC,QAAQ,GAAGJ;QACxCf,cAAc,cAAc;QAC5B,MAAMoB,WAAWN,KAAK,GAAG,KAAKD;QAC9BJ,UAAU,CAAC,4BAA4B,EAAES,kBAAkB;QAC3DT,UAAU,CAAC,kBAAkB,EAAEU,OAAO,OAAO,EAAE;QAC/CpB,cAAc;YACZ,MAAM;YACN,MAAMQ,qBAAqB,YAAY;YACvC,OAAO;gBACL,SAASY,OAAO,OAAO;gBACvBD;gBACA,OAAOC,OAAO,QAAQ;gBACtB,WAAWA,OAAO,SAAS;gBAC3BC;gBACA,mBAAmBD,OAAO,iBAAiB;YAC7C;QACF;QACA,OAAO;YACL,GAAGA,MAAM;YACT,mBAAmBD;YACnB,YAAYb;YACZe;QACF;IACF,EAAE,OAAOC,OAAY;QAGnB,IAAId,qBAAqB,YAAY,EAAE,IACrCR,cAAc;YACZ,MAAM;YACN,GAAGQ,qBAAqB,YAAY;YACpC,OAAOc,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QACzD;QAEFX,SAAS,WAAW;YAAE,SAAS;QAAK,GAAG,iBAAiBW;QACxD,IAAIA,iBAAiBG,sBACnB,MAAMH;QAER,MAAM,IAAIC,MACR,CAAC,eAAe,EAAEjB,cAAc,eAAe,GAAG,kBAAkB,EAAED,UAAU,GAAG,EAAEiB,MAAM,OAAO,GAAGI,4BAA4BJ,OAAOd,sBAAsB,8DAA8D,CAAC,EAC7N;YAAE,OAAOc;QAAM;IAEnB;AACF"}

package/dist/es/ai-model/service-caller/system-one.mjs

@@ -0,0 +1,163 @@
+import { assert, uuid } from "@midscene/shared/utils";
+import { z } from "zod";
+import { createProxyAgentIfNeeded } from "./proxy/index.mjs";
+const probability = z.number().min(0).max(1);
+const systemOneAnswerSchema = z.discriminatedUnion('type', [
+    z.object({
+        type: z.literal('noul'),
+        noul: probability
+    }),
+    z.object({
+        type: z.literal('choice'),
+        choice: z.string()
+    }),
+    z.object({
+        type: z.literal('box'),
+        box: z.object({
+            left: probability,
+            right: probability,
+            top: probability,
+            bottom: probability
+        }).refine((box)=>box.left < box.right && box.top < box.bottom, 'Invalid box edges')
+    })
+]);
+const systemOneResponseSchema = z.object({
+    model: z.string().optional(),
+    answers: z.record(systemOneAnswerSchema),
+    usage: z.object({
+        input_tokens: z.number().nonnegative().nullable().optional(),
+        output_tokens: z.number().nonnegative().nullable().optional()
+    }).optional()
+});
+async function prepareSystemOne(config, request) {
+    assert(config.openaiBaseURL, 'Jev requires MIDSCENE_*_MODEL_BASE_URL');
+    assert(config.openaiApiKey, 'Jev requires MIDSCENE_*_MODEL_API_KEY');
+    assert(!config.protocol || 'openai-chat' === config.protocol, 'Jev uses System One, not the Responses API');
+    const base = config.openaiBaseURL.replace(/\/+$/, '');
+    const url = `${base}${base.endsWith('/v1') ? '' : '/v1'}/systemone`;
+    const isOpenRouter = 'openrouter.ai' === new URL(base).hostname;
+    const assets = {};
+    const imageState = Object.fromEntries(Object.entries(request.images ?? {}).map(([name, dataURL])=>{
+        if (isOpenRouter) return [
+            name,
+            {
+                type: 'image_url',
+                image_url: {
+                    url: dataURL
+                }
+            }
+        ];
+        const reference = uuid();
+        const match = /^data:image\/(png|jpeg);base64,(.+)$/.exec(dataURL);
+        assert(match, 'System One images must be prepared PNG or JPEG data URLs');
+        assets[reference] = {
+            type: 'image',
+            format: 'png' === match[1] ? 'png' : 'jpg',
+            data_base64: match[2]
+        };
+        return [
+            name,
+            reference
+        ];
+    }));
+    const body = {
+        ...config.extraBody,
+        model: config.modelName,
+        state: {
+            ...request.state,
+            ...imageState
+        },
+        questions: request.questions,
+        ...!isOpenRouter && Object.keys(assets).length ? {
+            assets
+        } : {}
+    };
+    const serializedBody = JSON.stringify(body);
+    assert(new TextEncoder().encode(serializedBody).length <= 33554432, 'Jev request exceeds the 32 MiB body limit');
+    return {
+        url,
+        body: serializedBody,
+        questions: request.questions,
+        headers: {
+            ...config.openaiExtraConfig?.defaultHeaders,
+            Authorization: `Bearer ${config.openaiApiKey}`,
+            'Content-Type': 'application/json',
+            'TypeSafe-Danger-Zone-Images': 'true'
+        },
+        dispatcher: await createProxyAgentIfNeeded({
+            socksProxy: config.socksProxy,
+            httpProxy: config.httpProxy
+        })
+    };
+}
+async function callSystemOne(context, prepared) {
+    assert(!context.options?.stream, 'Jev does not support streaming output');
+    const start = Date.now();
+    context.recordEvent?.({
+        type: 'request',
+        request: {
+            url: prepared.url,
+            method: 'POST',
+            body: prepared.body
+        }
+    });
+    try {
+        const response = await fetch(prepared.url, {
+            method: 'POST',
+            headers: prepared.headers,
+            body: prepared.body,
+            signal: context.requestSignal,
+            ...prepared.dispatcher ? {
+                dispatcher: prepared.dispatcher
+            } : {}
+        });
+        const body = await response.text();
+        const http = {
+            status: response.status,
+            ok: response.ok,
+            body
+        };
+        if (!response.ok) {
+            context.recordEvent?.({
+                type: 'error',
+                http
+            });
+            throw new Error(`Jev request failed (${response.status}): ${body.slice(0, 4000)}`);
+        }
+        const result = systemOneResponseSchema.parse(JSON.parse(body));
+        for (const [name, question] of Object.entries(prepared.questions))assert(result.answers[name]?.type === question.type, `Missing or invalid Jev answer: ${name}`);
+        for (const [name, question] of Object.entries(prepared.questions)){
+            const answer = result.answers[name];
+            if ('choice' === question.type && 'choice' === answer.type) assert(Object.hasOwn(question.criteria, answer.choice), `Invalid Jev choice: ${name}`);
+        }
+        const input = result.usage?.input_tokens;
+        const output = result.usage?.output_tokens;
+        const final = {
+            content: body,
+            isStreamed: false,
+            rawUsage: null != input && null != output ? {
+                prompt_tokens: input,
+                completion_tokens: output,
+                total_tokens: input + output
+            } : void 0,
+            timeCost: Date.now() - start,
+            requestId: response.headers.get('x-request-id'),
+            responseModelName: result.model
+        };
+        context.recordEvent?.({
+            type: 'response',
+            http,
+            final
+        });
+        return final;
+    } catch (error) {
+        context.recordEvent?.({
+            type: 'error',
+            error: error instanceof Error ? error.message : String(error)
+        });
+        throw error;
+    }
+}
+export { callSystemOne, prepareSystemOne, systemOneAnswerSchema, systemOneResponseSchema };
+
+//# sourceMappingURL=system-one.mjs.map

package/dist/es/ai-model/service-caller/system-one.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/service-caller/system-one.mjs","sources":["../../../../src/ai-model/service-caller/system-one.ts"],"sourcesContent":["import type { IModelConfig } from '@midscene/shared/env';\nimport { assert, uuid } from '@midscene/shared/utils';\nimport { z } from 'zod';\nimport { createProxyAgentIfNeeded } from './proxy';\nimport type { ModelCallContext, ModelCallResult } from './types';\n\nexport interface SystemOneRequest {\n  state: Record<string, unknown>;\n  images?: Record<string, string>;\n  questions: Record<\n    string,\n    | { type: 'noul' | 'box'; instructions: unknown }\n    | {\n        type: 'choice';\n        instructions: unknown;\n        criteria: Record<string, string>;\n      }\n  >;\n}\n\nconst probability = z.number().min(0).max(1);\nexport const systemOneAnswerSchema = z.discriminatedUnion('type', [\n  z.object({ type: z.literal('noul'), noul: probability }),\n  z.object({ type: z.literal('choice'), choice: z.string() }),\n  z.object({\n    type: z.literal('box'),\n    box: z\n      .object({\n        left: probability,\n        right: probability,\n        top: probability,\n        bottom: probability,\n      })\n      .refine(\n        (box) => box.left < box.right && box.top < box.bottom,\n        'Invalid box edges',\n      ),\n  }),\n]);\nexport const systemOneResponseSchema = z.object({\n  model: z.string().optional(),\n  answers: z.record(systemOneAnswerSchema),\n  usage: z\n    .object({\n      input_tokens: z.number().nonnegative().nullable().optional(),\n      output_tokens: z.number().nonnegative().nullable().optional(),\n    })\n    .optional(),\n});\n\nexport async function prepareSystemOne(\n  config: IModelConfig,\n  request: SystemOneRequest,\n) {\n  assert(config.openaiBaseURL, 'Jev requires MIDSCENE_*_MODEL_BASE_URL');\n  assert(config.openaiApiKey, 'Jev requires MIDSCENE_*_MODEL_API_KEY');\n  assert(\n    !config.protocol || config.protocol === 'openai-chat',\n    'Jev uses System One, not the Responses API',\n  );\n  const base = config.openaiBaseURL.replace(/\\/+$/, '');\n  const url = `${base}${base.endsWith('/v1') ? '' : '/v1'}/systemone`;\n  const isOpenRouter = new URL(base).hostname === 'openrouter.ai';\n  const assets: Record<string, unknown> = {};\n  const imageState = Object.fromEntries(\n    Object.entries(request.images ?? {}).map(([name, dataURL]) => {\n      if (isOpenRouter) {\n        return [name, { type: 'image_url', image_url: { url: dataURL } }];\n      }\n      const reference = uuid();\n      const match = /^data:image\\/(png|jpeg);base64,(.+)$/.exec(dataURL);\n      assert(match, 'System One images must be prepared PNG or JPEG data URLs');\n      assets[reference] = {\n        type: 'image',\n        format: match[1] === 'png' ? 'png' : 'jpg',\n        data_base64: match[2],\n      };\n      return [name, reference];\n    }),\n  );\n  const body = {\n    ...config.extraBody,\n    model: config.modelName,\n    state: { ...request.state, ...imageState },\n    questions: request.questions,\n    ...(!isOpenRouter && Object.keys(assets).length ? { assets } : {}),\n  };\n  const serializedBody = JSON.stringify(body);\n  assert(\n    new TextEncoder().encode(serializedBody).length <= 32 * 1024 * 1024,\n    'Jev request exceeds the 32 MiB body limit',\n  );\n  return {\n    url,\n    body: serializedBody,\n    questions: request.questions,\n    headers: {\n      ...(config.openaiExtraConfig?.defaultHeaders as\n        | Record<string, string>\n        | undefined),\n      Authorization: `Bearer ${config.openaiApiKey}`,\n      'Content-Type': 'application/json',\n      'TypeSafe-Danger-Zone-Images': 'true',\n    },\n    dispatcher: await createProxyAgentIfNeeded({\n      socksProxy: config.socksProxy,\n      httpProxy: config.httpProxy,\n    }),\n  };\n}\n\nexport async function callSystemOne(\n  context: ModelCallContext,\n  prepared: Awaited<ReturnType<typeof prepareSystemOne>>,\n): Promise<ModelCallResult> {\n  assert(!context.options?.stream, 'Jev does not support streaming output');\n  const start = Date.now();\n  context.recordEvent?.({\n    type: 'request',\n    request: { url: prepared.url, method: 'POST', body: prepared.body },\n  });\n  try {\n    const response = await fetch(prepared.url, {\n      method: 'POST',\n      headers: prepared.headers,\n      body: prepared.body,\n      signal: context.requestSignal,\n      ...(prepared.dispatcher ? { dispatcher: prepared.dispatcher } : {}),\n    });\n    const body = await response.text();\n    const http = { status: response.status, ok: response.ok, body };\n    if (!response.ok) {\n      context.recordEvent?.({ type: 'error', http });\n      throw new Error(\n        `Jev request failed (${response.status}): ${body.slice(0, 4000)}`,\n      );\n    }\n    const result = systemOneResponseSchema.parse(JSON.parse(body));\n    for (const [name, question] of Object.entries(prepared.questions)) {\n      assert(\n        result.answers[name]?.type === question.type,\n        `Missing or invalid Jev answer: ${name}`,\n      );\n    }\n    for (const [name, question] of Object.entries(prepared.questions)) {\n      const answer = result.answers[name];\n      if (question.type === 'choice' && answer.type === 'choice') {\n        assert(\n          Object.hasOwn(question.criteria, answer.choice),\n          `Invalid Jev choice: ${name}`,\n        );\n      }\n    }\n    const input = result.usage?.input_tokens;\n    const output = result.usage?.output_tokens;\n    const final: ModelCallResult = {\n      content: body,\n      isStreamed: false,\n      rawUsage:\n        input != null && output != null\n          ? {\n              prompt_tokens: input,\n              completion_tokens: output,\n              total_tokens: input + output,\n            }\n          : undefined,\n      timeCost: Date.now() - start,\n      requestId: response.headers.get('x-request-id'),\n      responseModelName: result.model,\n    };\n    context.recordEvent?.({ type: 'response', http, final });\n    return final;\n  } catch (error) {\n    context.recordEvent?.({\n      type: 'error',\n      error: error instanceof Error ? error.message : String(error),\n    });\n    throw error;\n  }\n}\n"],"names":["probability","z","systemOneAnswerSchema","box","systemOneResponseSchema","prepareSystemOne","config","request","assert","base","url","isOpenRouter","URL","assets","imageState","Object","name","dataURL","reference","uuid","match","body","serializedBody","JSON","TextEncoder","createProxyAgentIfNeeded","callSystemOne","context","prepared","start","Date","response","fetch","http","Error","result","question","answer","input","output","final","undefined","error","String"],"mappings":";;;AAoBA,MAAMA,cAAcC,EAAE,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AACnC,MAAMC,wBAAwBD,EAAE,kBAAkB,CAAC,QAAQ;IAChEA,EAAE,MAAM,CAAC;QAAE,MAAMA,EAAE,OAAO,CAAC;QAAS,MAAMD;IAAY;IACtDC,EAAE,MAAM,CAAC;QAAE,MAAMA,EAAE,OAAO,CAAC;QAAW,QAAQA,EAAE,MAAM;IAAG;IACzDA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,OAAO,CAAC;QAChB,KAAKA,EAAAA,MACI,CAAC;YACN,MAAMD;YACN,OAAOA;YACP,KAAKA;YACL,QAAQA;QACV,GACC,MAAM,CACL,CAACG,MAAQA,IAAI,IAAI,GAAGA,IAAI,KAAK,IAAIA,IAAI,GAAG,GAAGA,IAAI,MAAM,EACrD;IAEN;CACD;AACM,MAAMC,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,OAAOA,EAAE,MAAM,GAAG,QAAQ;IAC1B,SAASA,EAAE,MAAM,CAACC;IAClB,OAAOD,EAAAA,MACE,CAAC;QACN,cAAcA,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ;QAC1D,eAAeA,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,QAAQ;IAC7D,GACC,QAAQ;AACb;AAEO,eAAeI,iBACpBC,MAAoB,EACpBC,OAAyB;IAEzBC,OAAOF,OAAO,aAAa,EAAE;IAC7BE,OAAOF,OAAO,YAAY,EAAE;IAC5BE,OACE,CAACF,OAAO,QAAQ,IAAIA,AAAoB,kBAApBA,OAAO,QAAQ,EACnC;IAEF,MAAMG,OAAOH,OAAO,aAAa,CAAC,OAAO,CAAC,QAAQ;IAClD,MAAMI,MAAM,GAAGD,OAAOA,KAAK,QAAQ,CAAC,SAAS,KAAK,MAAM,UAAU,CAAC;IACnE,MAAME,eAAe,AAA2B,oBAA3B,IAAIC,IAAIH,MAAM,QAAQ;IAC3C,MAAMI,SAAkC,CAAC;IACzC,MAAMC,aAAaC,OAAO,WAAW,CACnCA,OAAO,OAAO,CAACR,QAAQ,MAAM,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAACS,MAAMC,QAAQ;QACvD,IAAIN,cACF,OAAO;YAACK;YAAM;gBAAE,MAAM;gBAAa,WAAW;oBAAE,KAAKC;gBAAQ;YAAE;SAAE;QAEnE,MAAMC,YAAYC;QAClB,MAAMC,QAAQ,uCAAuC,IAAI,CAACH;QAC1DT,OAAOY,OAAO;QACdP,MAAM,CAACK,UAAU,GAAG;YAClB,MAAM;YACN,QAAQE,AAAa,UAAbA,KAAK,CAAC,EAAE,GAAa,QAAQ;YACrC,aAAaA,KAAK,CAAC,EAAE;QACvB;QACA,OAAO;YAACJ;YAAME;SAAU;IAC1B;IAEF,MAAMG,OAAO;QACX,GAAGf,OAAO,SAAS;QACnB,OAAOA,OAAO,SAAS;QACvB,OAAO;YAAE,GAAGC,QAAQ,KAAK;YAAE,GAAGO,UAAU;QAAC;QACzC,WAAWP,QAAQ,SAAS;QAC5B,GAAI,CAACI,gBAAgBI,OAAO,IAAI,CAACF,QAAQ,MAAM,GAAG;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnE;IACA,MAAMS,iBAAiBC,KAAK,SAAS,CAACF;IACtCb,OACE,IAAIgB,cAAc,MAAM,CAACF,gBAAgB,MAAM,IAAI,UACnD;IAEF,OAAO;QACLZ;QACA,MAAMY;QACN,WAAWf,QAAQ,SAAS;QAC5B,SAAS;YACP,GAAID,OAAO,iBAAiB,EAAE,cAAc;YAG5C,eAAe,CAAC,OAAO,EAAEA,OAAO,YAAY,EAAE;YAC9C,gBAAgB;YAChB,+BAA+B;QACjC;QACA,YAAY,MAAMmB,yBAAyB;YACzC,YAAYnB,OAAO,UAAU;YAC7B,WAAWA,OAAO,SAAS;QAC7B;IACF;AACF;AAEO,eAAeoB,cACpBC,OAAyB,EACzBC,QAAsD;IAEtDpB,OAAO,CAACmB,QAAQ,OAAO,EAAE,QAAQ;IACjC,MAAME,QAAQC,KAAK,GAAG;IACtBH,QAAQ,WAAW,GAAG;QACpB,MAAM;QACN,SAAS;YAAE,KAAKC,SAAS,GAAG;YAAE,QAAQ;YAAQ,MAAMA,SAAS,IAAI;QAAC;IACpE;IACA,IAAI;QACF,MAAMG,WAAW,MAAMC,MAAMJ,SAAS,GAAG,EAAE;YACzC,QAAQ;YACR,SAASA,SAAS,OAAO;YACzB,MAAMA,SAAS,IAAI;YACnB,QAAQD,QAAQ,aAAa;YAC7B,GAAIC,SAAS,UAAU,GAAG;gBAAE,YAAYA,SAAS,UAAU;YAAC,IAAI,CAAC,CAAC;QACpE;QACA,MAAMP,OAAO,MAAMU,SAAS,IAAI;QAChC,MAAME,OAAO;YAAE,QAAQF,SAAS,MAAM;YAAE,IAAIA,SAAS,EAAE;YAAEV;QAAK;QAC9D,IAAI,CAACU,SAAS,EAAE,EAAE;YAChBJ,QAAQ,WAAW,GAAG;gBAAE,MAAM;gBAASM;YAAK;YAC5C,MAAM,IAAIC,MACR,CAAC,oBAAoB,EAAEH,SAAS,MAAM,CAAC,GAAG,EAAEV,KAAK,KAAK,CAAC,GAAG,OAAO;QAErE;QACA,MAAMc,SAAS/B,wBAAwB,KAAK,CAACmB,KAAK,KAAK,CAACF;QACxD,KAAK,MAAM,CAACL,MAAMoB,SAAS,IAAIrB,OAAO,OAAO,CAACa,SAAS,SAAS,EAC9DpB,OACE2B,OAAO,OAAO,CAACnB,KAAK,EAAE,SAASoB,SAAS,IAAI,EAC5C,CAAC,+BAA+B,EAAEpB,MAAM;QAG5C,KAAK,MAAM,CAACA,MAAMoB,SAAS,IAAIrB,OAAO,OAAO,CAACa,SAAS,SAAS,EAAG;YACjE,MAAMS,SAASF,OAAO,OAAO,CAACnB,KAAK;YACnC,IAAIoB,AAAkB,aAAlBA,SAAS,IAAI,IAAiBC,AAAgB,aAAhBA,OAAO,IAAI,EAC3C7B,OACEO,OAAO,MAAM,CAACqB,SAAS,QAAQ,EAAEC,OAAO,MAAM,GAC9C,CAAC,oBAAoB,EAAErB,MAAM;QAGnC;QACA,MAAMsB,QAAQH,OAAO,KAAK,EAAE;QAC5B,MAAMI,SAASJ,OAAO,KAAK,EAAE;QAC7B,MAAMK,QAAyB;YAC7B,SAASnB;YACT,YAAY;YACZ,UACEiB,AAAS,QAATA,SAAiBC,AAAU,QAAVA,SACb;gBACE,eAAeD;gBACf,mBAAmBC;gBACnB,cAAcD,QAAQC;YACxB,IACAE;YACN,UAAUX,KAAK,GAAG,KAAKD;YACvB,WAAWE,SAAS,OAAO,CAAC,GAAG,CAAC;YAChC,mBAAmBI,OAAO,KAAK;QACjC;QACAR,QAAQ,WAAW,GAAG;YAAE,MAAM;YAAYM;YAAMO;QAAM;QACtD,OAAOA;IACT,EAAE,OAAOE,OAAO;QACdf,QAAQ,WAAW,GAAG;YACpB,MAAM;YACN,OAAOe,iBAAiBR,QAAQQ,MAAM,OAAO,GAAGC,OAAOD;QACzD;QACA,MAAMA;IACR;AACF"}

package/dist/es/ai-model/shared/model-locate-result/errors.mjs

@@ -1,9 +1,13 @@
 const modelConfigDocUrl = 'https://midscenejs.com/model-config';
-const defaultModelFamilyRequiredForLocateMessage = `Default model family is required for locate. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse locate coordinates correctly. ${modelConfigDocUrl}`;
+function modelFamilyRequiredForLocateMessage(slot) {
+    const role = 'locate' === slot ? 'Locate' : 'Default';
+    const configKey = 'locate' === slot ? 'MIDSCENE_LOCATE_MODEL_FAMILY' : 'MIDSCENE_MODEL_FAMILY';
+    return `${role} model family is required for locate. Configure ${configKey} so Midscene can parse locate coordinates correctly. ${modelConfigDocUrl}`;
+}
 function planningModelFamilyRequiredForLocateMessage(slot) {
     if ('planning' === slot) return `Planning model family is required because aiAct is asking the planning model to return locate coordinates. Configure MIDSCENE_PLANNING_MODEL_FAMILY for the planning model, or remove the separate planning model config and configure MIDSCENE_MODEL_FAMILY on the default model. ${modelConfigDocUrl}`;
     return `Default model family is required because aiAct is asking the default model to return locate coordinates during planning. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse planning locate coordinates correctly. ${modelConfigDocUrl}`;
 }
-export { defaultModelFamilyRequiredForLocateMessage, planningModelFamilyRequiredForLocateMessage };
+export { modelFamilyRequiredForLocateMessage, planningModelFamilyRequiredForLocateMessage };
 
 //# sourceMappingURL=errors.mjs.map

package/dist/es/ai-model/shared/model-locate-result/errors.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/shared/model-locate-result/errors.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/errors.ts"],"sourcesContent":["const modelConfigDocUrl = 'https://midscenejs.com/model-config';\n\nexport const defaultModelFamilyRequiredForLocateMessage = `Default model family is required for locate. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse locate coordinates correctly. ${modelConfigDocUrl}`;\n\nexport function planningModelFamilyRequiredForLocateMessage(slot?: string) {\n  if (slot === 'planning') {\n    return `Planning model family is required because aiAct is asking the planning model to return locate coordinates. Configure MIDSCENE_PLANNING_MODEL_FAMILY for the planning model, or remove the separate planning model config and configure MIDSCENE_MODEL_FAMILY on the default model. ${modelConfigDocUrl}`;\n  }\n\n  return `Default model family is required because aiAct is asking the default model to return locate coordinates during planning. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse planning locate coordinates correctly. ${modelConfigDocUrl}`;\n}\n"],"names":["modelConfigDocUrl","defaultModelFamilyRequiredForLocateMessage","planningModelFamilyRequiredForLocateMessage","slot"],"mappings":"AAAA,MAAMA,oBAAoB;AAEnB,MAAMC,6CAA6C,CAAC,iIAAiI,EAAED,mBAAmB;AAE1M,SAASE,4CAA4CC,IAAa;IACvE,IAAIA,AAAS,eAATA,MACF,OAAO,CAAC,mRAAmR,EAAEH,mBAAmB;IAGlT,OAAO,CAAC,sNAAsN,EAAEA,mBAAmB;AACrP"}
+{"version":3,"file":"ai-model/shared/model-locate-result/errors.mjs","sources":["../../../../../src/ai-model/shared/model-locate-result/errors.ts"],"sourcesContent":["const modelConfigDocUrl = 'https://midscenejs.com/model-config';\n\nexport function modelFamilyRequiredForLocateMessage(slot?: string) {\n  const role = slot === 'locate' ? 'Locate' : 'Default';\n  const configKey =\n    slot === 'locate'\n      ? 'MIDSCENE_LOCATE_MODEL_FAMILY'\n      : 'MIDSCENE_MODEL_FAMILY';\n  return `${role} model family is required for locate. Configure ${configKey} so Midscene can parse locate coordinates correctly. ${modelConfigDocUrl}`;\n}\n\nexport function planningModelFamilyRequiredForLocateMessage(slot?: string) {\n  if (slot === 'planning') {\n    return `Planning model family is required because aiAct is asking the planning model to return locate coordinates. Configure MIDSCENE_PLANNING_MODEL_FAMILY for the planning model, or remove the separate planning model config and configure MIDSCENE_MODEL_FAMILY on the default model. ${modelConfigDocUrl}`;\n  }\n\n  return `Default model family is required because aiAct is asking the default model to return locate coordinates during planning. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse planning locate coordinates correctly. ${modelConfigDocUrl}`;\n}\n"],"names":["modelConfigDocUrl","modelFamilyRequiredForLocateMessage","slot","role","configKey","planningModelFamilyRequiredForLocateMessage"],"mappings":"AAAA,MAAMA,oBAAoB;AAEnB,SAASC,oCAAoCC,IAAa;IAC/D,MAAMC,OAAOD,AAAS,aAATA,OAAoB,WAAW;IAC5C,MAAME,YACJF,AAAS,aAATA,OACI,iCACA;IACN,OAAO,GAAGC,KAAK,gDAAgD,EAAEC,UAAU,qDAAqD,EAAEJ,mBAAmB;AACvJ;AAEO,SAASK,4CAA4CH,IAAa;IACvE,IAAIA,AAAS,eAATA,MACF,OAAO,CAAC,mRAAmR,EAAEF,mBAAmB;IAGlT,OAAO,CAAC,sNAAsN,EAAEA,mBAAmB;AACrP"}

package/dist/es/ai-model/workflows/insight/boolean.mjs

@@ -0,0 +1,20 @@
+import { assert } from "@midscene/shared/utils";
+async function extractBoolean(options) {
+    const { booleanFn } = options.modelRuntime.adapter.insight;
+    assert(booleanFn, 'A boolean adapter is required');
+    const keys = 'object' == typeof options.dataQuery ? Object.keys(options.dataQuery) : [];
+    assert(1 === keys.length && ('Boolean' === keys[0] || 'StatementIsTruthy' === keys[0]), 'This model only supports boolean insight tasks. Use the default model for extraction.');
+    const result = await booleanFn(options);
+    return {
+        parseResult: {
+            data: {
+                [keys[0]]: result.value
+            }
+        },
+        rawResponse: result.rawResponse,
+        usage: result.usage
+    };
+}
+export { extractBoolean };
+
+//# sourceMappingURL=boolean.mjs.map

package/dist/es/ai-model/workflows/insight/boolean.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/insight/boolean.mjs","sources":["../../../../../src/ai-model/workflows/insight/boolean.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIUsageInfo,\n  RawAssistantOutput,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\nimport type { ExtractOptions } from './extraction';\n\nexport type BooleanFn = (options: ExtractOptions) => Promise<{\n  value: boolean;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}>;\n\nexport async function extractBoolean<T>(options: ExtractOptions): Promise<{\n  parseResult: AIDataExtractionResponse<T>;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n  rawAssistantOutput?: RawAssistantOutput;\n  reasoning_content?: string;\n}> {\n  const { booleanFn } = options.modelRuntime.adapter.insight;\n  assert(booleanFn, 'A boolean adapter is required');\n  const keys =\n    typeof options.dataQuery === 'object' ? Object.keys(options.dataQuery) : [];\n  assert(\n    keys.length === 1 &&\n      (keys[0] === 'Boolean' || keys[0] === 'StatementIsTruthy'),\n    'This model only supports boolean insight tasks. Use the default model for extraction.',\n  );\n  const result = await booleanFn(options);\n  return {\n    parseResult: {\n      data: { [keys[0]]: result.value } as T,\n    },\n    rawResponse: result.rawResponse,\n    usage: result.usage,\n  };\n}\n"],"names":["extractBoolean","options","booleanFn","assert","keys","Object","result"],"mappings":";AAcO,eAAeA,eAAkBC,OAAuB;IAO7D,MAAM,EAAEC,SAAS,EAAE,GAAGD,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO;IAC1DE,OAAOD,WAAW;IAClB,MAAME,OACJ,AAA6B,YAA7B,OAAOH,QAAQ,SAAS,GAAgBI,OAAO,IAAI,CAACJ,QAAQ,SAAS,IAAI,EAAE;IAC7EE,OACEC,AAAgB,MAAhBA,KAAK,MAAM,IACRA,CAAAA,AAAY,cAAZA,IAAI,CAAC,EAAE,IAAkBA,AAAY,wBAAZA,IAAI,CAAC,EAAE,AAAuB,GAC1D;IAEF,MAAME,SAAS,MAAMJ,UAAUD;IAC/B,OAAO;QACL,aAAa;YACX,MAAM;gBAAE,CAACG,IAAI,CAAC,EAAE,CAAC,EAAEE,OAAO,KAAK;YAAC;QAClC;QACA,aAAaA,OAAO,WAAW;QAC/B,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/workflows/insight/extraction.mjs

@@ -4,9 +4,11 @@ import { buildInsightSystemPrompt, extractDataQueryPrompt } from "../../prompt/i
 import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
 import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
 import { multimodalPromptToMessages } from "../../shared/multimodal-prompt.mjs";
+import { extractBoolean } from "./boolean.mjs";
 import { parseInsightResponse } from "./insight-response-parser.mjs";
 const debugInsight = getDebug('ai:insight');
 async function AiExtractElementInfo(options) {
+    if (options.modelRuntime.adapter.insight.booleanFn) return extractBoolean(options);
     const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } = options;
     const insightProtocol = modelRuntime.adapter.insight.protocol;
     const systemPrompt = buildInsightSystemPrompt({

package/dist/es/ai-model/workflows/insight/extraction.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/insight/extraction.mjs","sources":["../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n} from '@/ai-model/service-caller/types';\nimport type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [ConversationSystemMessage, ...ConversationUserMessage[]];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ConversationUserMessage['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image',\n          url: frame.base64,\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image',\n        url: screenshotBase64,\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawAssistantOutput,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawAssistantOutput,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["debugInsight","getDebug","AiExtractElementInfo","options","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawAssistantOutput","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;;;;;AAuBA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,qBAAwBC,OAQ7C;IACC,MAAM,EAAEC,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEL;IACF,MAAMM,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,yBAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,gBAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,uBAC5Bb,QAAQ,eAAe,IAAI,IAC3BC,WACAS;IAGF,MAAMI,cAAkD,EAAE;IAE1D,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,KAAKE,MAAM,MAAM;gBACnB;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,KAAKL;YACP;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,2BAA2BhB;QAChDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaL,QAAQ,WAAW;gBAChC,sBAAsBsB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,GAAGJ;YACJ,MAAMK,cAAcC,qBAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB;QAE/B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaL,QAAQ,WAAW;QAChC,cAAc,CAACsC;YACbzC,aACE,iDACAyC,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/insight/extraction.mjs","sources":["../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["import type {\n  ConversationSystemMessage,\n  ConversationUserMessage,\n} from '@/ai-model/service-caller/types';\nimport type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { renderAIContext } from '../../../agent/prompt-context';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToMessages } from '../../shared/multimodal-prompt';\nimport { extractBoolean } from './boolean';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [ConversationSystemMessage, ...ConversationUserMessage[]];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport interface ExtractOptions {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}\n\nexport async function AiExtractElementInfo<T>(options: ExtractOptions) {\n  if (options.modelRuntime.adapter.insight.booleanFn) {\n    return extractBoolean<T>(options);\n  }\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const renderedContext = renderAIContext(extractOption?.context);\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    renderedContext,\n  );\n\n  const userContent: ConversationUserMessage['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image',\n          url: frame.base64,\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image',\n        url: screenshotBase64,\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawAssistantOutput,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawAssistantOutput,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawAssistantOutput,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["debugInsight","getDebug","AiExtractElementInfo","options","extractBoolean","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","renderedContext","renderAIContext","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawAssistantOutput","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;;;;;;AAwBA,MAAMA,eAAeC,SAAS;AAYvB,eAAeC,qBAAwBC,OAAuB;IACnE,IAAIA,QAAQ,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAChD,OAAOC,eAAkBD;IAE3B,MAAM,EAAEE,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEN;IACF,MAAMO,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,yBAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMQ,kBAAkBC,gBAAgBR,eAAe;IACvD,MAAMS,wBAAwBC,uBAC5Bd,QAAQ,eAAe,IAAI,IAC3BE,WACAS;IAGF,MAAMI,cAAkD,EAAE;IAE1D,IAAIX,eAAe,uBAAuB,OAAO;QAC/C,MAAMY,qBAAqBb,QAAQ,kBAAkB;QACrD,IAAIa,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,KAAKE,MAAM,MAAM;gBACnB;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,KAAKL;YACP;QACF;IACF;IAEAK,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASX;QAAa;QACxC;YACE,MAAM;YACN,SAASO;QACX;KACD;IAED,IAAIV,kBAAkB;QACpB,MAAMe,SAAS,MAAMC,2BAA2BhB;QAChDc,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,MAAMK,qBAChClB,cACA;gBACE,aAAaN,QAAQ,WAAW;gBAChC,sBAAsBuB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,GAAGJ;YACJ,MAAMK,cAAcC,qBAClBL,aACArB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACL0B;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB;QAE/B;QACA,iBAAiBrB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaN,QAAQ,WAAW;QAChC,cAAc,CAACuC;YACb1C,aACE,iDACA0C,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}