@midscene/core 0.30.10 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent/agent.mjs +233 -144
- package/dist/es/agent/agent.mjs.map +1 -1
- package/dist/es/agent/execution-session.mjs +41 -0
- package/dist/es/agent/execution-session.mjs.map +1 -0
- package/dist/es/agent/index.mjs +3 -3
- package/dist/es/agent/task-builder.mjs +319 -0
- package/dist/es/agent/task-builder.mjs.map +1 -0
- package/dist/es/agent/task-cache.mjs +4 -4
- package/dist/es/agent/task-cache.mjs.map +1 -1
- package/dist/es/agent/tasks.mjs +197 -504
- package/dist/es/agent/tasks.mjs.map +1 -1
- package/dist/es/agent/ui-utils.mjs +54 -35
- package/dist/es/agent/ui-utils.mjs.map +1 -1
- package/dist/es/agent/utils.mjs +16 -58
- package/dist/es/agent/utils.mjs.map +1 -1
- package/dist/es/ai-model/conversation-history.mjs +25 -13
- package/dist/es/ai-model/conversation-history.mjs.map +1 -1
- package/dist/es/ai-model/index.mjs +4 -4
- package/dist/es/ai-model/inspect.mjs +45 -54
- package/dist/es/ai-model/inspect.mjs.map +1 -1
- package/dist/es/ai-model/llm-planning.mjs +47 -65
- package/dist/es/ai-model/llm-planning.mjs.map +1 -1
- package/dist/es/ai-model/prompt/assertion.mjs.map +1 -1
- package/dist/es/ai-model/prompt/common.mjs.map +1 -1
- package/dist/es/ai-model/prompt/describe.mjs.map +1 -1
- package/dist/es/ai-model/prompt/extraction.mjs.map +1 -1
- package/dist/es/ai-model/prompt/llm-locator.mjs +11 -235
- package/dist/es/ai-model/prompt/llm-locator.mjs.map +1 -1
- package/dist/es/ai-model/prompt/llm-planning.mjs +76 -322
- package/dist/es/ai-model/prompt/llm-planning.mjs.map +1 -1
- package/dist/es/ai-model/prompt/llm-section-locator.mjs +15 -14
- package/dist/es/ai-model/prompt/llm-section-locator.mjs.map +1 -1
- package/dist/es/ai-model/prompt/order-sensitive-judge.mjs +35 -0
- package/dist/es/ai-model/prompt/order-sensitive-judge.mjs.map +1 -0
- package/dist/es/ai-model/prompt/playwright-generator.mjs +2 -2
- package/dist/es/ai-model/prompt/playwright-generator.mjs.map +1 -1
- package/dist/es/ai-model/prompt/ui-tars-locator.mjs.map +1 -1
- package/dist/es/ai-model/prompt/ui-tars-planning.mjs.map +1 -1
- package/dist/es/ai-model/prompt/util.mjs +3 -88
- package/dist/es/ai-model/prompt/util.mjs.map +1 -1
- package/dist/es/ai-model/prompt/yaml-generator.mjs +10 -10
- package/dist/es/ai-model/prompt/yaml-generator.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/index.mjs +182 -274
- package/dist/es/ai-model/service-caller/index.mjs.map +1 -1
- package/dist/es/ai-model/ui-tars-planning.mjs +69 -8
- package/dist/es/ai-model/ui-tars-planning.mjs.map +1 -1
- package/dist/es/{ai-model/common.mjs → common.mjs} +18 -30
- package/dist/es/common.mjs.map +1 -0
- package/dist/es/device/device-options.mjs +0 -0
- package/dist/es/device/index.mjs +29 -12
- package/dist/es/device/index.mjs.map +1 -1
- package/dist/es/index.mjs +5 -4
- package/dist/es/index.mjs.map +1 -1
- package/dist/es/report.mjs.map +1 -1
- package/dist/es/{insight → service}/index.mjs +38 -51
- package/dist/es/service/index.mjs.map +1 -0
- package/dist/es/{insight → service}/utils.mjs +3 -3
- package/dist/es/service/utils.mjs.map +1 -0
- package/dist/es/task-runner.mjs +264 -0
- package/dist/es/task-runner.mjs.map +1 -0
- package/dist/es/tree.mjs +13 -2
- package/dist/es/tree.mjs.map +1 -0
- package/dist/es/types.mjs +18 -1
- package/dist/es/types.mjs.map +1 -1
- package/dist/es/utils.mjs +6 -7
- package/dist/es/utils.mjs.map +1 -1
- package/dist/es/yaml/builder.mjs.map +1 -1
- package/dist/es/yaml/player.mjs +121 -98
- package/dist/es/yaml/player.mjs.map +1 -1
- package/dist/es/yaml/utils.mjs +1 -1
- package/dist/es/yaml/utils.mjs.map +1 -1
- package/dist/lib/agent/agent.js +231 -142
- package/dist/lib/agent/agent.js.map +1 -1
- package/dist/lib/agent/common.js +1 -1
- package/dist/lib/agent/execution-session.js +75 -0
- package/dist/lib/agent/execution-session.js.map +1 -0
- package/dist/lib/agent/index.js +14 -14
- package/dist/lib/agent/index.js.map +1 -1
- package/dist/lib/agent/task-builder.js +356 -0
- package/dist/lib/agent/task-builder.js.map +1 -0
- package/dist/lib/agent/task-cache.js +8 -8
- package/dist/lib/agent/task-cache.js.map +1 -1
- package/dist/lib/agent/tasks.js +202 -506
- package/dist/lib/agent/tasks.js.map +1 -1
- package/dist/lib/agent/ui-utils.js +58 -36
- package/dist/lib/agent/ui-utils.js.map +1 -1
- package/dist/lib/agent/utils.js +26 -68
- package/dist/lib/agent/utils.js.map +1 -1
- package/dist/lib/ai-model/conversation-history.js +27 -15
- package/dist/lib/ai-model/conversation-history.js.map +1 -1
- package/dist/lib/ai-model/index.js +27 -27
- package/dist/lib/ai-model/index.js.map +1 -1
- package/dist/lib/ai-model/inspect.js +51 -57
- package/dist/lib/ai-model/inspect.js.map +1 -1
- package/dist/lib/ai-model/llm-planning.js +49 -67
- package/dist/lib/ai-model/llm-planning.js.map +1 -1
- package/dist/lib/ai-model/prompt/assertion.js +2 -2
- package/dist/lib/ai-model/prompt/assertion.js.map +1 -1
- package/dist/lib/ai-model/prompt/common.js +2 -2
- package/dist/lib/ai-model/prompt/common.js.map +1 -1
- package/dist/lib/ai-model/prompt/describe.js +2 -2
- package/dist/lib/ai-model/prompt/describe.js.map +1 -1
- package/dist/lib/ai-model/prompt/extraction.js +2 -2
- package/dist/lib/ai-model/prompt/extraction.js.map +1 -1
- package/dist/lib/ai-model/prompt/llm-locator.js +14 -241
- package/dist/lib/ai-model/prompt/llm-locator.js.map +1 -1
- package/dist/lib/ai-model/prompt/llm-planning.js +79 -328
- package/dist/lib/ai-model/prompt/llm-planning.js.map +1 -1
- package/dist/lib/ai-model/prompt/llm-section-locator.js +17 -16
- package/dist/lib/ai-model/prompt/llm-section-locator.js.map +1 -1
- package/dist/lib/ai-model/prompt/order-sensitive-judge.js +72 -0
- package/dist/lib/ai-model/prompt/order-sensitive-judge.js.map +1 -0
- package/dist/lib/ai-model/prompt/playwright-generator.js +11 -11
- package/dist/lib/ai-model/prompt/playwright-generator.js.map +1 -1
- package/dist/lib/ai-model/prompt/ui-tars-locator.js +2 -2
- package/dist/lib/ai-model/prompt/ui-tars-locator.js.map +1 -1
- package/dist/lib/ai-model/prompt/ui-tars-planning.js +2 -2
- package/dist/lib/ai-model/prompt/ui-tars-planning.js.map +1 -1
- package/dist/lib/ai-model/prompt/util.js +7 -95
- package/dist/lib/ai-model/prompt/util.js.map +1 -1
- package/dist/lib/ai-model/prompt/yaml-generator.js +18 -18
- package/dist/lib/ai-model/prompt/yaml-generator.js.map +1 -1
- package/dist/lib/ai-model/service-caller/index.js +288 -401
- package/dist/lib/ai-model/service-caller/index.js.map +1 -1
- package/dist/lib/ai-model/ui-tars-planning.js +71 -10
- package/dist/lib/ai-model/ui-tars-planning.js.map +1 -1
- package/dist/lib/{ai-model/common.js → common.js} +40 -55
- package/dist/lib/common.js.map +1 -0
- package/dist/lib/device/device-options.js +20 -0
- package/dist/lib/device/device-options.js.map +1 -0
- package/dist/lib/device/index.js +63 -40
- package/dist/lib/device/index.js.map +1 -1
- package/dist/lib/image/index.js +5 -5
- package/dist/lib/image/index.js.map +1 -1
- package/dist/lib/index.js +24 -20
- package/dist/lib/index.js.map +1 -1
- package/dist/lib/report.js +2 -2
- package/dist/lib/report.js.map +1 -1
- package/dist/lib/{insight → service}/index.js +41 -54
- package/dist/lib/service/index.js.map +1 -0
- package/dist/lib/{insight → service}/utils.js +7 -7
- package/dist/lib/service/utils.js.map +1 -0
- package/dist/lib/task-runner.js +301 -0
- package/dist/lib/task-runner.js.map +1 -0
- package/dist/lib/tree.js +13 -4
- package/dist/lib/tree.js.map +1 -1
- package/dist/lib/types.js +31 -12
- package/dist/lib/types.js.map +1 -1
- package/dist/lib/utils.js +16 -17
- package/dist/lib/utils.js.map +1 -1
- package/dist/lib/yaml/builder.js +2 -2
- package/dist/lib/yaml/builder.js.map +1 -1
- package/dist/lib/yaml/index.js +16 -22
- package/dist/lib/yaml/index.js.map +1 -1
- package/dist/lib/yaml/player.js +123 -100
- package/dist/lib/yaml/player.js.map +1 -1
- package/dist/lib/yaml/utils.js +6 -6
- package/dist/lib/yaml/utils.js.map +1 -1
- package/dist/lib/yaml.js +1 -1
- package/dist/lib/yaml.js.map +1 -1
- package/dist/types/agent/agent.d.ts +62 -17
- package/dist/types/agent/execution-session.d.ts +36 -0
- package/dist/types/agent/index.d.ts +3 -2
- package/dist/types/agent/task-builder.d.ts +35 -0
- package/dist/types/agent/tasks.d.ts +32 -23
- package/dist/types/agent/ui-utils.d.ts +9 -2
- package/dist/types/agent/utils.d.ts +9 -35
- package/dist/types/ai-model/conversation-history.d.ts +8 -4
- package/dist/types/ai-model/index.d.ts +5 -5
- package/dist/types/ai-model/inspect.d.ts +20 -12
- package/dist/types/ai-model/llm-planning.d.ts +3 -1
- package/dist/types/ai-model/prompt/llm-locator.d.ts +1 -6
- package/dist/types/ai-model/prompt/llm-planning.d.ts +2 -3
- package/dist/types/ai-model/prompt/llm-section-locator.d.ts +1 -3
- package/dist/types/ai-model/prompt/order-sensitive-judge.d.ts +2 -0
- package/dist/types/ai-model/prompt/util.d.ts +2 -34
- package/dist/types/ai-model/service-caller/index.d.ts +2 -3
- package/dist/types/ai-model/ui-tars-planning.d.ts +15 -2
- package/dist/types/{ai-model/common.d.ts → common.d.ts} +6 -6
- package/dist/types/device/device-options.d.ts +57 -0
- package/dist/types/device/index.d.ts +55 -39
- package/dist/types/index.d.ts +7 -6
- package/dist/types/service/index.d.ts +26 -0
- package/dist/types/service/utils.d.ts +2 -0
- package/dist/types/task-runner.d.ts +49 -0
- package/dist/types/tree.d.ts +4 -1
- package/dist/types/types.d.ts +103 -66
- package/dist/types/yaml/utils.d.ts +1 -1
- package/dist/types/yaml.d.ts +68 -43
- package/package.json +9 -12
- package/dist/es/ai-model/action-executor.mjs +0 -129
- package/dist/es/ai-model/action-executor.mjs.map +0 -1
- package/dist/es/ai-model/common.mjs.map +0 -1
- package/dist/es/insight/index.mjs.map +0 -1
- package/dist/es/insight/utils.mjs.map +0 -1
- package/dist/lib/ai-model/action-executor.js +0 -163
- package/dist/lib/ai-model/action-executor.js.map +0 -1
- package/dist/lib/ai-model/common.js.map +0 -1
- package/dist/lib/insight/index.js.map +0 -1
- package/dist/lib/insight/utils.js.map +0 -1
- package/dist/types/ai-model/action-executor.d.ts +0 -19
- package/dist/types/insight/index.d.ts +0 -31
- package/dist/types/insight/utils.d.ts +0 -2
package/dist/lib/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sources":["webpack
|
|
1
|
+
{"version":3,"file":"index.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/index.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { z } from 'zod';\nimport Service from './service/index';\nimport { TaskRunner } from './task-runner';\nimport { getVersion } from './utils';\n\nexport {\n plan,\n describeUserPage,\n AiLocateElement,\n getMidsceneLocationSchema,\n type MidsceneLocationResultType,\n PointSchema,\n SizeSchema,\n RectSchema,\n TMultimodalPromptSchema,\n TUserPromptSchema,\n type TMultimodalPrompt,\n type TUserPrompt,\n} from './ai-model/index';\n\nexport {\n MIDSCENE_MODEL_NAME,\n type CreateOpenAIClientFn,\n} from '@midscene/shared/env';\n\nexport type * from './types';\nexport { ServiceError } from './types';\n\nexport { z };\n\nexport default Service;\nexport { TaskRunner, Service, getVersion };\n\nexport type {\n MidsceneYamlScript,\n MidsceneYamlTask,\n MidsceneYamlFlowItem,\n MidsceneYamlConfigResult,\n MidsceneYamlConfig,\n MidsceneYamlScriptWebEnv,\n MidsceneYamlScriptAndroidEnv,\n MidsceneYamlScriptIOSEnv,\n MidsceneYamlScriptEnv,\n LocateOption,\n DetailedLocateParam,\n} from './yaml';\n\nexport { Agent, type AgentOpt, type AiActOptions, createAgent } from './agent';\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","Service"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACwBA,YAAeI"}
|
package/dist/lib/report.js
CHANGED
|
@@ -112,9 +112,9 @@ class ReportMergingTool {
|
|
|
112
112
|
}
|
|
113
113
|
}
|
|
114
114
|
exports.ReportMergingTool = __webpack_exports__.ReportMergingTool;
|
|
115
|
-
for(var
|
|
115
|
+
for(var __rspack_i in __webpack_exports__)if (-1 === [
|
|
116
116
|
"ReportMergingTool"
|
|
117
|
-
].indexOf(
|
|
117
|
+
].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
|
|
118
118
|
Object.defineProperty(exports, '__esModule', {
|
|
119
119
|
value: true
|
|
120
120
|
});
|
package/dist/lib/report.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"report.js","sources":["webpack
|
|
1
|
+
{"version":3,"file":"report.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/report.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import * as fs from 'node:fs';\nimport * as path from 'node:path';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { getReportFileName } from './agent';\nimport type { ReportFileWithAttributes } from './types';\nimport { getReportTpl, reportHTMLContent } from './utils';\n\nexport class ReportMergingTool {\n private reportInfos: ReportFileWithAttributes[] = [];\n public append(reportInfo: ReportFileWithAttributes) {\n this.reportInfos.push(reportInfo);\n }\n public clear() {\n this.reportInfos = [];\n }\n private extractScriptContent(filePath: string): string {\n // Regular expression to match content between script tags\n // Requires newline before <script and </script>\n const scriptRegex =\n /\\n<script type=\"midscene_web_dump\" type=\"application\\/json\"[^>]*>([\\s\\S]*?)\\n<\\/script>/;\n\n const fileContent = fs.readFileSync(filePath, 'utf-8');\n const match = scriptRegex.exec(fileContent);\n\n return match ? match[1].trim() : '';\n }\n\n public mergeReports(\n reportFileName: 'AUTO' | string = 'AUTO', // user custom report filename, save into midscene report dir if undefined\n opts?: {\n rmOriginalReports?: boolean; // whether to remove origin report files\n overwrite?: boolean; // if output filepath specified, throw an error when overwrite = true, otherwise overwrite the file\n },\n ): string | null {\n if (this.reportInfos.length <= 1) {\n console.log('Not enough report to merge');\n return null;\n }\n opts = Object.assign(\n {\n rmOriginalReports: false,\n overwrite: false,\n },\n opts || {},\n );\n const { rmOriginalReports, overwrite } = opts;\n let outputFilePath;\n const targetDir = `${getMidsceneRunSubDir('report')}`;\n if (reportFileName === 'AUTO') {\n outputFilePath = path.resolve(\n targetDir,\n `${getReportFileName('merged-report')}.html`,\n );\n } else {\n // user specified a output filepath\n outputFilePath = path.resolve(targetDir, `${reportFileName}.html`);\n if (fs.existsSync(outputFilePath) && !overwrite) {\n throw Error(\n `report file already existed: ${outputFilePath}\\nset override to true to overwrite this file.`,\n );\n } else if (fs.existsSync(outputFilePath) && overwrite) {\n fs.unlinkSync(outputFilePath);\n }\n }\n\n console.log(\n `Start merging ${this.reportInfos.length} reports...\\nCreating template file...`,\n );\n\n try {\n // Write template\n fs.appendFileSync(outputFilePath, getReportTpl());\n\n // Process all reports one by one\n for (let i = 0; i < this.reportInfos.length; i++) {\n const reportInfo = this.reportInfos[i];\n console.log(`Processing report ${i + 1}/${this.reportInfos.length}`);\n\n const dumpString = this.extractScriptContent(reportInfo.reportFilePath);\n const reportAttributes = reportInfo.reportAttributes;\n\n const reportHtmlStr = `${reportHTMLContent(\n {\n dumpString,\n attributes: {\n playwright_test_duration: reportAttributes.testDuration,\n playwright_test_status: reportAttributes.testStatus,\n playwright_test_title: reportAttributes.testTitle,\n playwright_test_id: reportAttributes.testId,\n playwright_test_description: reportAttributes.testDescription,\n },\n },\n undefined,\n undefined,\n false,\n )}\\n`; // use existed function to achieve report script content\n\n fs.appendFileSync(outputFilePath, reportHtmlStr);\n }\n\n console.log(`Successfully merged new report: ${outputFilePath}`);\n\n // Remove original reports if needed\n if (rmOriginalReports) {\n for (const info of this.reportInfos) {\n try {\n fs.unlinkSync(info.reportFilePath);\n } catch (error) {\n console.error(\n `Error deleting report ${info.reportFilePath}:`,\n error,\n );\n }\n }\n console.log(`Removed ${this.reportInfos.length} original reports`);\n }\n return outputFilePath;\n } catch (error) {\n console.error('Error in mergeReports:', error);\n throw error;\n }\n }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","ReportMergingTool","reportInfo","filePath","scriptRegex","fileContent","fs","match","reportFileName","opts","console","rmOriginalReports","overwrite","outputFilePath","targetDir","getMidsceneRunSubDir","path","getReportFileName","Error","getReportTpl","i","dumpString","reportAttributes","reportHtmlStr","reportHTMLContent","undefined","info","error"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;ACCO,MAAMI;IAEJ,OAAOC,UAAoC,EAAE;QAClD,IAAI,CAAC,WAAW,CAAC,IAAI,CAACA;IACxB;IACO,QAAQ;QACb,IAAI,CAAC,WAAW,GAAG,EAAE;IACvB;IACQ,qBAAqBC,QAAgB,EAAU;QAGrD,MAAMC,cACJ;QAEF,MAAMC,cAAcC,iCAAAA,YAAe,CAACH,UAAU;QAC9C,MAAMI,QAAQH,YAAY,IAAI,CAACC;QAE/B,OAAOE,QAAQA,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK;IACnC;IAEO,aACLC,iBAAkC,MAAM,EACxCC,IAGC,EACc;QACf,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,GAAG;YAChCC,QAAQ,GAAG,CAAC;YACZ,OAAO;QACT;QACAD,OAAOZ,OAAO,MAAM,CAClB;YACE,mBAAmB;YACnB,WAAW;QACb,GACAY,QAAQ,CAAC;QAEX,MAAM,EAAEE,iBAAiB,EAAEC,SAAS,EAAE,GAAGH;QACzC,IAAII;QACJ,MAAMC,YAAY,GAAGC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WAAW;QACrD,IAAIP,AAAmB,WAAnBA,gBACFK,iBAAiBG,mCAAAA,OAAY,CAC3BF,WACA,GAAGG,AAAAA,IAAAA,yBAAAA,iBAAAA,AAAAA,EAAkB,iBAAiB,KAAK,CAAC;aAEzC;YAELJ,iBAAiBG,mCAAAA,OAAY,CAACF,WAAW,GAAGN,eAAe,KAAK,CAAC;YACjE,IAAIF,iCAAAA,UAAa,CAACO,mBAAmB,CAACD,WACpC,MAAMM,MACJ,CAAC,6BAA6B,EAAEL,eAAe,8CAA8C,CAAC;YAE3F,IAAIP,iCAAAA,UAAa,CAACO,mBAAmBD,WAC1CN,iCAAAA,UAAa,CAACO;QAElB;QAEAH,QAAQ,GAAG,CACT,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,sCAAsC,CAAC;QAGlF,IAAI;YAEFJ,iCAAAA,cAAiB,CAACO,gBAAgBM,AAAAA,IAAAA,kCAAAA,YAAAA,AAAAA;YAGlC,IAAK,IAAIC,IAAI,GAAGA,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAEA,IAAK;gBAChD,MAAMlB,aAAa,IAAI,CAAC,WAAW,CAACkB,EAAE;gBACtCV,QAAQ,GAAG,CAAC,CAAC,kBAAkB,EAAEU,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;gBAEnE,MAAMC,aAAa,IAAI,CAAC,oBAAoB,CAACnB,WAAW,cAAc;gBACtE,MAAMoB,mBAAmBpB,WAAW,gBAAgB;gBAEpD,MAAMqB,gBAAgB,GAAGC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EACvB;oBACEH;oBACA,YAAY;wBACV,0BAA0BC,iBAAiB,YAAY;wBACvD,wBAAwBA,iBAAiB,UAAU;wBACnD,uBAAuBA,iBAAiB,SAAS;wBACjD,oBAAoBA,iBAAiB,MAAM;wBAC3C,6BAA6BA,iBAAiB,eAAe;oBAC/D;gBACF,GACAG,QACAA,QACA,OACA,EAAE,CAAC;gBAELnB,iCAAAA,cAAiB,CAACO,gBAAgBU;YACpC;YAEAb,QAAQ,GAAG,CAAC,CAAC,gCAAgC,EAAEG,gBAAgB;YAG/D,IAAIF,mBAAmB;gBACrB,KAAK,MAAMe,QAAQ,IAAI,CAAC,WAAW,CACjC,IAAI;oBACFpB,iCAAAA,UAAa,CAACoB,KAAK,cAAc;gBACnC,EAAE,OAAOC,OAAO;oBACdjB,QAAQ,KAAK,CACX,CAAC,sBAAsB,EAAEgB,KAAK,cAAc,CAAC,CAAC,CAAC,EAC/CC;gBAEJ;gBAEFjB,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,iBAAiB,CAAC;YACnE;YACA,OAAOG;QACT,EAAE,OAAOc,OAAO;YACdjB,QAAQ,KAAK,CAAC,0BAA0BiB;YACxC,MAAMA;QACR;IACF;;QAjHA,uBAAQ,eAA0C,EAAE;;AAkHtD"}
|
|
@@ -24,12 +24,13 @@ var __webpack_require__ = {};
|
|
|
24
24
|
var __webpack_exports__ = {};
|
|
25
25
|
__webpack_require__.r(__webpack_exports__);
|
|
26
26
|
__webpack_require__.d(__webpack_exports__, {
|
|
27
|
-
default: ()=>
|
|
27
|
+
default: ()=>Service
|
|
28
28
|
});
|
|
29
|
-
const common_js_namespaceObject = require("../ai-model/common.js");
|
|
30
29
|
const index_js_namespaceObject = require("../ai-model/index.js");
|
|
31
30
|
const inspect_js_namespaceObject = require("../ai-model/inspect.js");
|
|
32
31
|
const describe_js_namespaceObject = require("../ai-model/prompt/describe.js");
|
|
32
|
+
const external_common_js_namespaceObject = require("../common.js");
|
|
33
|
+
const external_types_js_namespaceObject = require("../types.js");
|
|
33
34
|
const env_namespaceObject = require("@midscene/shared/env");
|
|
34
35
|
const img_namespaceObject = require("@midscene/shared/img");
|
|
35
36
|
const logger_namespaceObject = require("@midscene/shared/logger");
|
|
@@ -45,14 +46,11 @@ function _define_property(obj, key, value) {
|
|
|
45
46
|
else obj[key] = value;
|
|
46
47
|
return obj;
|
|
47
48
|
}
|
|
48
|
-
const debug = (0, logger_namespaceObject.getDebug)('ai:
|
|
49
|
-
class
|
|
49
|
+
const debug = (0, logger_namespaceObject.getDebug)('ai:service');
|
|
50
|
+
class Service {
|
|
50
51
|
async locate(query, opt, modelConfig) {
|
|
51
|
-
var _parseResult_errors;
|
|
52
52
|
const queryPrompt = 'string' == typeof query ? query : query.prompt;
|
|
53
53
|
(0, utils_namespaceObject.assert)(queryPrompt, 'query is required for locate');
|
|
54
|
-
const dumpSubscriber = this.onceDumpUpdatedFn;
|
|
55
|
-
this.onceDumpUpdatedFn = void 0;
|
|
56
54
|
(0, utils_namespaceObject.assert)('object' == typeof query, 'query should be an object for locate');
|
|
57
55
|
const globalDeepThinkSwitch = env_namespaceObject.globalConfigManager.getEnvConfigInBoolean(env_namespaceObject.MIDSCENE_FORCE_DEEP_THINK);
|
|
58
56
|
if (globalDeepThinkSwitch) debug('globalDeepThinkSwitch', globalDeepThinkSwitch);
|
|
@@ -60,10 +58,10 @@ class Insight {
|
|
|
60
58
|
if (query.deepThink || globalDeepThinkSwitch) searchAreaPrompt = query.prompt;
|
|
61
59
|
const { vlMode } = modelConfig;
|
|
62
60
|
if (searchAreaPrompt && !vlMode) {
|
|
63
|
-
console.warn('The "deepThink" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/
|
|
61
|
+
console.warn('The "deepThink" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config');
|
|
64
62
|
searchAreaPrompt = void 0;
|
|
65
63
|
}
|
|
66
|
-
const context =
|
|
64
|
+
const context = opt?.context || await this.contextRetrieverFn();
|
|
67
65
|
let searchArea;
|
|
68
66
|
let searchAreaRawResponse;
|
|
69
67
|
let searchAreaUsage;
|
|
@@ -80,7 +78,7 @@ class Insight {
|
|
|
80
78
|
searchArea = searchAreaResponse.rect;
|
|
81
79
|
}
|
|
82
80
|
const startTime = Date.now();
|
|
83
|
-
const { parseResult, rect,
|
|
81
|
+
const { parseResult, rect, rawResponse, usage } = await (0, index_js_namespaceObject.AiLocateElement)({
|
|
84
82
|
callAIFn: this.aiVendorFn,
|
|
85
83
|
context,
|
|
86
84
|
targetElementDescription: queryPrompt,
|
|
@@ -99,7 +97,7 @@ class Insight {
|
|
|
99
97
|
searchAreaUsage
|
|
100
98
|
};
|
|
101
99
|
let errorLog;
|
|
102
|
-
if (
|
|
100
|
+
if (parseResult.errors?.length) errorLog = `failed to locate element: \n${parseResult.errors.join('\n')}`;
|
|
103
101
|
const dumpData = {
|
|
104
102
|
type: 'locate',
|
|
105
103
|
userQuery: {
|
|
@@ -112,50 +110,39 @@ class Insight {
|
|
|
112
110
|
deepThink: !!searchArea,
|
|
113
111
|
error: errorLog
|
|
114
112
|
};
|
|
115
|
-
const elements = [];
|
|
116
|
-
(
|
|
117
|
-
if ('id' in item) {
|
|
118
|
-
const element = elementById(null == item ? void 0 : item.id);
|
|
119
|
-
if (!element) return void console.warn(`locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`);
|
|
120
|
-
elements.push(element);
|
|
121
|
-
}
|
|
122
|
-
});
|
|
123
|
-
(0, external_utils_js_namespaceObject.emitInsightDump)({
|
|
113
|
+
const elements = parseResult.elements || [];
|
|
114
|
+
const dump = (0, external_utils_js_namespaceObject.createServiceDump)({
|
|
124
115
|
...dumpData,
|
|
125
116
|
matchedElement: elements
|
|
126
|
-
}
|
|
127
|
-
if (errorLog) throw new
|
|
128
|
-
|
|
117
|
+
});
|
|
118
|
+
if (errorLog) throw new external_types_js_namespaceObject.ServiceError(errorLog, dump);
|
|
119
|
+
if (elements.length > 1) throw new external_types_js_namespaceObject.ServiceError(`locate: multiple elements found, length = ${elements.length}`, dump);
|
|
129
120
|
if (1 === elements.length) return {
|
|
130
121
|
element: {
|
|
131
|
-
id: elements[0].id,
|
|
132
|
-
indexId: elements[0].indexId,
|
|
133
122
|
center: elements[0].center,
|
|
134
123
|
rect: elements[0].rect,
|
|
135
|
-
|
|
136
|
-
attributes: elements[0].attributes,
|
|
137
|
-
isOrderSensitive
|
|
124
|
+
description: elements[0].description
|
|
138
125
|
},
|
|
139
|
-
rect
|
|
126
|
+
rect,
|
|
127
|
+
dump
|
|
140
128
|
};
|
|
141
129
|
return {
|
|
142
130
|
element: null,
|
|
143
|
-
rect
|
|
131
|
+
rect,
|
|
132
|
+
dump
|
|
144
133
|
};
|
|
145
134
|
}
|
|
146
|
-
async extract(dataDemand, modelConfig, opt, multimodalPrompt) {
|
|
147
|
-
var _parseResult_errors;
|
|
135
|
+
async extract(dataDemand, modelConfig, opt, pageDescription, multimodalPrompt) {
|
|
148
136
|
(0, utils_namespaceObject.assert)('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
|
|
149
|
-
const
|
|
150
|
-
this.onceDumpUpdatedFn = void 0;
|
|
151
|
-
const context = await this.contextRetrieverFn('extract');
|
|
137
|
+
const context = await this.contextRetrieverFn();
|
|
152
138
|
const startTime = Date.now();
|
|
153
139
|
const { parseResult, usage } = await (0, index_js_namespaceObject.AiExtractElementInfo)({
|
|
154
140
|
context,
|
|
155
141
|
dataQuery: dataDemand,
|
|
156
142
|
multimodalPrompt,
|
|
157
143
|
extractOption: opt,
|
|
158
|
-
modelConfig
|
|
144
|
+
modelConfig,
|
|
145
|
+
pageDescription
|
|
159
146
|
});
|
|
160
147
|
const timeCost = Date.now() - startTime;
|
|
161
148
|
const taskInfo = {
|
|
@@ -164,7 +151,7 @@ class Insight {
|
|
|
164
151
|
rawResponse: JSON.stringify(parseResult)
|
|
165
152
|
};
|
|
166
153
|
let errorLog;
|
|
167
|
-
if (
|
|
154
|
+
if (parseResult.errors?.length) errorLog = `AI response error: \n${parseResult.errors.join('\n')}`;
|
|
168
155
|
const dumpData = {
|
|
169
156
|
type: 'extract',
|
|
170
157
|
userQuery: {
|
|
@@ -176,22 +163,23 @@ class Insight {
|
|
|
176
163
|
error: errorLog
|
|
177
164
|
};
|
|
178
165
|
const { data, thought } = parseResult || {};
|
|
179
|
-
(0, external_utils_js_namespaceObject.
|
|
166
|
+
const dump = (0, external_utils_js_namespaceObject.createServiceDump)({
|
|
180
167
|
...dumpData,
|
|
181
168
|
data
|
|
182
|
-
}
|
|
183
|
-
if (errorLog && !data
|
|
169
|
+
});
|
|
170
|
+
if (errorLog && !data) throw new external_types_js_namespaceObject.ServiceError(errorLog, dump);
|
|
184
171
|
return {
|
|
185
172
|
data,
|
|
186
173
|
thought,
|
|
187
|
-
usage
|
|
174
|
+
usage,
|
|
175
|
+
dump
|
|
188
176
|
};
|
|
189
177
|
}
|
|
190
178
|
async describe(target, modelConfig, opt) {
|
|
191
|
-
(0, utils_namespaceObject.assert)(target, 'target is required for
|
|
192
|
-
const context = await this.contextRetrieverFn(
|
|
179
|
+
(0, utils_namespaceObject.assert)(target, 'target is required for service.describe');
|
|
180
|
+
const context = await this.contextRetrieverFn();
|
|
193
181
|
const { screenshotBase64, size } = context;
|
|
194
|
-
(0, utils_namespaceObject.assert)(screenshotBase64, 'screenshot is required for
|
|
182
|
+
(0, utils_namespaceObject.assert)(screenshotBase64, 'screenshot is required for service.describe');
|
|
195
183
|
const { vlMode } = modelConfig;
|
|
196
184
|
const systemPrompt = (0, describe_js_namespaceObject.elementDescriberInstruction)();
|
|
197
185
|
const defaultRectSize = 30;
|
|
@@ -211,10 +199,10 @@ class Insight {
|
|
|
211
199
|
],
|
|
212
200
|
borderThickness: 3
|
|
213
201
|
});
|
|
214
|
-
if (
|
|
215
|
-
const searchArea = (0,
|
|
202
|
+
if (opt?.deepThink) {
|
|
203
|
+
const searchArea = (0, external_common_js_namespaceObject.expandSearchArea)(targetRect, context.size, vlMode);
|
|
216
204
|
debug('describe: set searchArea', searchArea);
|
|
217
|
-
const croppedResult = await (0, img_namespaceObject.cropByRect)(imagePayload, searchArea, '
|
|
205
|
+
const croppedResult = await (0, img_namespaceObject.cropByRect)(imagePayload, searchArea, 'qwen2.5-vl' === vlMode);
|
|
218
206
|
imagePayload = croppedResult.imageBase64;
|
|
219
207
|
}
|
|
220
208
|
const msgs = [
|
|
@@ -236,7 +224,7 @@ class Insight {
|
|
|
236
224
|
}
|
|
237
225
|
];
|
|
238
226
|
const callAIFn = this.aiVendorFn;
|
|
239
|
-
const res = await callAIFn(msgs,
|
|
227
|
+
const res = await callAIFn(msgs, external_common_js_namespaceObject.AIActionType.DESCRIBE_ELEMENT, modelConfig);
|
|
240
228
|
const { content } = res;
|
|
241
229
|
(0, utils_namespaceObject.assert)(!content.error, `describe failed: ${content.error}`);
|
|
242
230
|
(0, utils_namespaceObject.assert)(content.description, 'failed to describe the element');
|
|
@@ -245,19 +233,18 @@ class Insight {
|
|
|
245
233
|
constructor(context, opt){
|
|
246
234
|
_define_property(this, "contextRetrieverFn", void 0);
|
|
247
235
|
_define_property(this, "aiVendorFn", index_js_namespaceObject.callAIWithObjectResponse);
|
|
248
|
-
_define_property(this, "onceDumpUpdatedFn", void 0);
|
|
249
236
|
_define_property(this, "taskInfo", void 0);
|
|
250
|
-
(0, utils_namespaceObject.assert)(context, 'context is required for
|
|
237
|
+
(0, utils_namespaceObject.assert)(context, 'context is required for Service');
|
|
251
238
|
if ('function' == typeof context) this.contextRetrieverFn = context;
|
|
252
239
|
else this.contextRetrieverFn = ()=>Promise.resolve(context);
|
|
253
|
-
if (void 0 !==
|
|
254
|
-
if (void 0 !==
|
|
240
|
+
if (void 0 !== opt?.aiVendorFn) this.aiVendorFn = opt.aiVendorFn;
|
|
241
|
+
if (void 0 !== opt?.taskInfo) this.taskInfo = opt.taskInfo;
|
|
255
242
|
}
|
|
256
243
|
}
|
|
257
244
|
exports["default"] = __webpack_exports__["default"];
|
|
258
|
-
for(var
|
|
245
|
+
for(var __rspack_i in __webpack_exports__)if (-1 === [
|
|
259
246
|
"default"
|
|
260
|
-
].indexOf(
|
|
247
|
+
].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
|
|
261
248
|
Object.defineProperty(exports, '__esModule', {
|
|
262
249
|
value: true
|
|
263
250
|
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"service/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n AiExtractElementInfo,\n AiLocateElement,\n callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport { AIActionType, type AIArgs, expandSearchArea } from '@/common';\nimport type {\n AIDescribeElementResponse,\n AIUsageInfo,\n DetailedLocateParam,\n LocateResultWithDump,\n PartialServiceDumpFromSDK,\n Rect,\n ServiceExtractOption,\n ServiceExtractParam,\n ServiceExtractResult,\n ServiceTaskInfo,\n UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport {\n type IModelConfig,\n MIDSCENE_FORCE_DEEP_THINK,\n globalConfigManager,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n context?: UIContext;\n}\n\nexport type AnyValue<T> = {\n [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n aiVendorFn?: typeof callAIWithObjectResponse;\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n aiVendorFn: Exclude<ServiceOptions['aiVendorFn'], undefined> =\n callAIWithObjectResponse;\n\n taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n constructor(\n context: UIContext | (() => Promise<UIContext> | UIContext),\n opt?: ServiceOptions,\n ) {\n assert(context, 'context is required for Service');\n if (typeof context === 'function') {\n this.contextRetrieverFn = context;\n } else {\n this.contextRetrieverFn = () => Promise.resolve(context);\n }\n\n // just for unit test, aiVendorFn is callAIWithObjectResponse by default\n if (typeof opt?.aiVendorFn !== 'undefined') {\n this.aiVendorFn = opt.aiVendorFn;\n }\n if (typeof opt?.taskInfo !== 'undefined') {\n this.taskInfo = opt.taskInfo;\n }\n }\n\n async locate(\n query: DetailedLocateParam,\n opt: LocateOpts,\n modelConfig: IModelConfig,\n ): Promise<LocateResultWithDump> {\n const queryPrompt = typeof query === 'string' ? query : query.prompt;\n assert(queryPrompt, 'query is required for locate');\n\n assert(typeof query === 'object', 'query should be an object for locate');\n\n const globalDeepThinkSwitch = globalConfigManager.getEnvConfigInBoolean(\n MIDSCENE_FORCE_DEEP_THINK,\n );\n if (globalDeepThinkSwitch) {\n debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n }\n let searchAreaPrompt;\n if (query.deepThink || globalDeepThinkSwitch) {\n searchAreaPrompt = query.prompt;\n }\n\n const { vlMode } = modelConfig;\n\n if (searchAreaPrompt && !vlMode) {\n console.warn(\n 'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config',\n );\n searchAreaPrompt = undefined;\n }\n\n const context = opt?.context || (await this.contextRetrieverFn());\n\n let searchArea: Rect | undefined = undefined;\n let searchAreaRawResponse: string | undefined = undefined;\n let searchAreaUsage: AIUsageInfo | undefined = undefined;\n let searchAreaResponse:\n | Awaited<ReturnType<typeof AiLocateSection>>\n | undefined = undefined;\n if (searchAreaPrompt) {\n searchAreaResponse = await AiLocateSection({\n context,\n sectionDescription: searchAreaPrompt,\n modelConfig,\n });\n assert(\n searchAreaResponse.rect,\n `cannot find search area for \"${searchAreaPrompt}\"${\n searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n }`,\n );\n searchAreaRawResponse = searchAreaResponse.rawResponse;\n searchAreaUsage = searchAreaResponse.usage;\n searchArea = searchAreaResponse.rect;\n }\n\n const startTime = Date.now();\n const { parseResult, rect, rawResponse, usage } = await AiLocateElement({\n callAIFn: this.aiVendorFn,\n context,\n targetElementDescription: queryPrompt,\n searchConfig: searchAreaResponse,\n modelConfig,\n });\n\n const timeCost = Date.now() - startTime;\n const taskInfo: ServiceTaskInfo = {\n ...(this.taskInfo ? this.taskInfo : {}),\n durationMs: timeCost,\n rawResponse: JSON.stringify(rawResponse),\n formatResponse: JSON.stringify(parseResult),\n usage,\n searchArea,\n searchAreaRawResponse,\n searchAreaUsage,\n };\n\n let errorLog: string | undefined;\n if (parseResult.errors?.length) {\n errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n }\n\n const dumpData: PartialServiceDumpFromSDK = {\n type: 'locate',\n userQuery: {\n element: queryPrompt,\n },\n matchedElement: [],\n matchedRect: rect,\n data: null,\n taskInfo,\n deepThink: !!searchArea,\n error: errorLog,\n };\n\n const elements = parseResult.elements || [];\n\n const dump = createServiceDump({\n ...dumpData,\n matchedElement: elements,\n });\n\n if (errorLog) {\n throw new ServiceError(errorLog, dump);\n }\n\n if (elements.length > 1) {\n throw new ServiceError(\n `locate: multiple elements found, length = ${elements.length}`,\n dump,\n );\n }\n\n if (elements.length === 1) {\n return {\n element: {\n center: elements[0]!.center,\n rect: elements[0]!.rect,\n description: elements[0]!.description,\n },\n rect,\n dump,\n };\n }\n\n return {\n element: null,\n rect,\n dump,\n };\n }\n\n async extract<T>(\n dataDemand: ServiceExtractParam,\n modelConfig: IModelConfig,\n opt?: ServiceExtractOption,\n pageDescription?: string,\n multimodalPrompt?: TMultimodalPrompt,\n ): Promise<ServiceExtractResult<T>> {\n assert(\n typeof dataDemand === 'object' || typeof dataDemand === 'string',\n `dataDemand should be object or string, but get ${typeof dataDemand}`,\n );\n const context = await this.contextRetrieverFn();\n\n const startTime = Date.now();\n\n const { parseResult, usage } = await AiExtractElementInfo<T>({\n context,\n dataQuery: dataDemand,\n multimodalPrompt,\n extractOption: opt,\n modelConfig,\n pageDescription,\n });\n\n const timeCost = Date.now() - startTime;\n const taskInfo: ServiceTaskInfo = {\n ...(this.taskInfo ? this.taskInfo : {}),\n durationMs: timeCost,\n rawResponse: JSON.stringify(parseResult),\n };\n\n let errorLog: string | undefined;\n if (parseResult.errors?.length) {\n errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n }\n\n const dumpData: PartialServiceDumpFromSDK = {\n type: 'extract',\n userQuery: {\n dataDemand,\n },\n matchedElement: [],\n data: null,\n taskInfo,\n error: errorLog,\n };\n\n const { data, thought } = parseResult || {};\n\n // 4\n const dump = createServiceDump({\n ...dumpData,\n data,\n });\n\n if (errorLog && !data) {\n throw new ServiceError(errorLog, dump);\n }\n\n return {\n data,\n thought,\n usage,\n dump,\n };\n }\n\n async describe(\n target: Rect | [number, number],\n modelConfig: IModelConfig,\n opt?: {\n deepThink?: boolean;\n },\n ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n assert(target, 'target is required for service.describe');\n const context = await this.contextRetrieverFn();\n const { screenshotBase64, size } = context;\n assert(screenshotBase64, 'screenshot is required for service.describe');\n // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n const { vlMode } = modelConfig;\n const systemPrompt = elementDescriberInstruction();\n\n // Convert [x,y] center point to Rect if needed\n const defaultRectSize = 30;\n const targetRect: Rect = Array.isArray(target)\n ? {\n left: Math.floor(target[0] - defaultRectSize / 2),\n top: Math.floor(target[1] - defaultRectSize / 2),\n width: defaultRectSize,\n height: defaultRectSize,\n }\n : target;\n\n let imagePayload = await compositeElementInfoImg({\n inputImgBase64: screenshotBase64,\n size,\n elementsPositionInfo: [\n {\n rect: targetRect,\n },\n ],\n borderThickness: 3,\n });\n\n if (opt?.deepThink) {\n const searchArea = expandSearchArea(targetRect, context.size, vlMode);\n debug('describe: set searchArea', searchArea);\n const croppedResult = await cropByRect(\n imagePayload,\n searchArea,\n vlMode === 'qwen2.5-vl',\n );\n imagePayload = croppedResult.imageBase64;\n }\n\n const msgs: AIArgs = [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: [\n {\n type: 'image_url',\n image_url: {\n url: imagePayload,\n detail: 'high',\n },\n },\n ],\n },\n ];\n\n const callAIFn = this\n .aiVendorFn as typeof callAIWithObjectResponse<AIDescribeElementResponse>;\n\n const res = await callAIFn(\n msgs,\n AIActionType.DESCRIBE_ELEMENT,\n modelConfig,\n );\n\n const { content } = res;\n assert(!content.error, `describe failed: ${content.error}`);\n assert(content.description, 'failed to describe the element');\n return content;\n }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Service","query","opt","modelConfig","queryPrompt","assert","globalDeepThinkSwitch","globalConfigManager","MIDSCENE_FORCE_DEEP_THINK","searchAreaPrompt","vlMode","console","undefined","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","dump","createServiceDump","ServiceError","dataDemand","pageDescription","multimodalPrompt","AiExtractElementInfo","data","thought","target","screenshotBase64","size","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","callAIFn","res","AIActionType","content","callAIWithObjectResponse","Promise"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;ACwCA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACR,MAAMC;IA4BnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACM;QAC/B,MAAMC,cAAc,AAAiB,YAAjB,OAAOH,QAAqBA,QAAQA,MAAM,MAAM;QACpEI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QAEpBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAOJ,OAAoB;QAElC,MAAMK,wBAAwBC,oBAAAA,mBAAAA,CAAAA,qBAAyC,CACrEC,oBAAAA,yBAAyBA;QAE3B,IAAIF,uBACFR,MAAM,yBAAyBQ;QAEjC,IAAIG;QACJ,IAAIR,MAAM,SAAS,IAAIK,uBACrBG,mBAAmBR,MAAM,MAAM;QAGjC,MAAM,EAAES,MAAM,EAAE,GAAGP;QAEnB,IAAIM,oBAAoB,CAACC,QAAQ;YAC/BC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,MAAMC,UAAUX,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIY;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIR,kBAAkB;YACpBQ,qBAAqB,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBACzCL;gBACA,oBAAoBJ;gBACpBN;YACF;YACAE,IAAAA,sBAAAA,MAAAA,AAAAA,EACEY,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAER,iBAAiB,CAAC,EAChDQ,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAE,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgB;YACtE,UAAU,IAAI,CAAC,UAAU;YACzBZ;YACA,0BAA0BT;YAC1B,cAAca;YACdd;QACF;QAEA,MAAMuB,WAAWN,KAAK,GAAG,KAAKD;QAC9B,MAAMQ,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACL;YAC5B,gBAAgBK,KAAK,SAAS,CAACP;YAC/BG;YACAV;YACAC;YACAC;QACF;QAEA,IAAIa;QACJ,IAAIR,YAAY,MAAM,EAAE,QACtBQ,WAAW,CAAC,4BAA4B,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS1B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAakB;YACb,MAAM;YACNK;YACA,WAAW,CAAC,CAACb;YACb,OAAOe;QACT;QAEA,MAAME,WAAWV,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMW,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,kCAAAA,YAAYA,CACpB,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,aAAaA,QAAQ,CAAC,EAAE,CAAE,WAAW;YACvC;YACAT;YACAU;QACF;QAGF,OAAO;YACL,SAAS;YACTV;YACAU;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/BhC,WAAyB,EACzBD,GAA0B,EAC1BkC,eAAwB,EACxBC,gBAAoC,EACF;QAClChC,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAO8B,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAEvE,MAAMtB,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAE7C,MAAMM,YAAYC,KAAK,GAAG;QAE1B,MAAM,EAAEC,WAAW,EAAEG,KAAK,EAAE,GAAG,MAAMc,AAAAA,IAAAA,yBAAAA,oBAAAA,AAAAA,EAAwB;YAC3DzB;YACA,WAAWsB;YACXE;YACA,eAAenC;YACfC;YACAiC;QACF;QAEA,MAAMV,WAAWN,KAAK,GAAG,KAAKD;QAC9B,MAAMQ,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACP;QAC9B;QAEA,IAAIQ;QACJ,IAAIR,YAAY,MAAM,EAAE,QACtBQ,WAAW,CAAC,qBAAqB,EAAER,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMS,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNR;YACA,OAAOE;QACT;QAEA,MAAM,EAAEU,IAAI,EAAEC,OAAO,EAAE,GAAGnB,eAAe,CAAC;QAG1C,MAAMW,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACXS;QACF;QAEA,IAAIV,YAAY,CAACU,MACf,MAAM,IAAIL,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,OAAO;YACLO;YACAC;YACAhB;YACAQ;QACF;IACF;IAEA,MAAM,SACJS,MAA+B,EAC/BtC,WAAyB,EACzBD,GAEC,EACwD;QACzDG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOoC,QAAQ;QACf,MAAM5B,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAE6B,gBAAgB,EAAEC,IAAI,EAAE,GAAG9B;QACnCR,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOqC,kBAAkB;QAEzB,MAAM,EAAEhC,MAAM,EAAE,GAAGP;QACnB,MAAMyC,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBT;YAChBC;YACA,sBAAsB;gBACpB;oBACE,MAAMI;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAI7C,KAAK,WAAW;YAClB,MAAMY,aAAasC,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYlC,QAAQ,IAAI,EAAEH;YAC9DZ,MAAM,4BAA4BgB;YAClC,MAAMuC,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BJ,cACApC,YACAJ,AAAW,iBAAXA;YAEFwC,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,WAAW,IAAI,CAClB,UAAU;QAEb,MAAMC,MAAM,MAAMD,SAChBD,MACAG,mCAAAA,YAAAA,CAAAA,gBAA6B,EAC7BvD;QAGF,MAAM,EAAEwD,OAAO,EAAE,GAAGF;QACpBpD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACsD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DtD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOsD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAvSA,YACE9C,OAA2D,EAC3DX,GAAoB,CACpB;QAVF;QAEA,qCACE0D,yBAAAA,wBAAwBA;QAE1B;QAMEvD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOQ,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMgD,QAAQ,OAAO,CAAChD;QAIlD,IAAI,AAA2B,WAApBX,KAAK,YACd,IAAI,CAAC,UAAU,GAAGA,IAAI,UAAU;QAElC,IAAI,AAAyB,WAAlBA,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAsRF"}
|
|
@@ -24,10 +24,10 @@ var __webpack_require__ = {};
|
|
|
24
24
|
var __webpack_exports__ = {};
|
|
25
25
|
__webpack_require__.r(__webpack_exports__);
|
|
26
26
|
__webpack_require__.d(__webpack_exports__, {
|
|
27
|
-
|
|
27
|
+
createServiceDump: ()=>createServiceDump
|
|
28
28
|
});
|
|
29
29
|
const utils_namespaceObject = require("@midscene/shared/utils");
|
|
30
|
-
function
|
|
30
|
+
function createServiceDump(data) {
|
|
31
31
|
const baseData = {
|
|
32
32
|
logTime: Date.now()
|
|
33
33
|
};
|
|
@@ -36,12 +36,12 @@ function emitInsightDump(data, dumpSubscriber) {
|
|
|
36
36
|
...baseData,
|
|
37
37
|
...data
|
|
38
38
|
};
|
|
39
|
-
|
|
39
|
+
return finalData;
|
|
40
40
|
}
|
|
41
|
-
exports.
|
|
42
|
-
for(var
|
|
43
|
-
"
|
|
44
|
-
].indexOf(
|
|
41
|
+
exports.createServiceDump = __webpack_exports__.createServiceDump;
|
|
42
|
+
for(var __rspack_i in __webpack_exports__)if (-1 === [
|
|
43
|
+
"createServiceDump"
|
|
44
|
+
].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
|
|
45
45
|
Object.defineProperty(exports, '__esModule', {
|
|
46
46
|
value: true
|
|
47
47
|
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"service/utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { DumpMeta, PartialServiceDumpFromSDK, ServiceDump } from '@/types';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function createServiceDump(\n data: PartialServiceDumpFromSDK,\n): ServiceDump {\n const baseData: DumpMeta = {\n logTime: Date.now(),\n };\n const finalData: ServiceDump = {\n logId: uuid(),\n ...baseData,\n ...data,\n };\n\n return finalData;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","createServiceDump","data","baseData","Date","finalData","uuid"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACHO,SAASI,kBACdC,IAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA;QACP,GAAGH,QAAQ;QACX,GAAGD,IAAI;IACT;IAEA,OAAOG;AACT"}
|