@midscene/core 0.3.4 → 0.4.0

package/dist/lib/ai-model.js

@@ -1556,39 +1556,39 @@ var Client = class _Client {
     }
     return runParams;
   }
-  async _getResponse(path2, queryParams) {
+  async _getResponse(path, queryParams) {
     var _a;
     const paramsString = (_a = queryParams == null ? void 0 : queryParams.toString()) != null ? _a : "";
-    const url = `${this.apiUrl}${path2}?${paramsString}`;
+    const url = `${this.apiUrl}${path}?${paramsString}`;
     const response = await this.caller.call(fetch, url, __spreadValues({
       method: "GET",
       headers: this.headers,
       signal: AbortSignal.timeout(this.timeout_ms)
     }, this.fetchOptions));
     if (!response.ok) {
-      throw new Error(`Failed to fetch ${path2}: ${response.status} ${response.statusText}`);
+      throw new Error(`Failed to fetch ${path}: ${response.status} ${response.statusText}`);
     }
     return response;
   }
-  async _get(path2, queryParams) {
-    const response = await this._getResponse(path2, queryParams);
+  async _get(path, queryParams) {
+    const response = await this._getResponse(path, queryParams);
     return response.json();
   }
   _getPaginated(_0) {
-    return __asyncGenerator(this, arguments, function* (path2, queryParams = new URLSearchParams()) {
+    return __asyncGenerator(this, arguments, function* (path, queryParams = new URLSearchParams()) {
       let offset = Number(queryParams.get("offset")) || 0;
       const limit = Number(queryParams.get("limit")) || 100;
       while (true) {
         queryParams.set("offset", String(offset));
         queryParams.set("limit", String(limit));
-        const url = `${this.apiUrl}${path2}?${queryParams}`;
+        const url = `${this.apiUrl}${path}?${queryParams}`;
         const response = yield new __await(this.caller.call(fetch, url, __spreadValues({
           method: "GET",
           headers: this.headers,
           signal: AbortSignal.timeout(this.timeout_ms)
         }, this.fetchOptions)));
         if (!response.ok) {
-          throw new Error(`Failed to fetch ${path2}: ${response.status} ${response.statusText}`);
+          throw new Error(`Failed to fetch ${path}: ${response.status} ${response.statusText}`);
         }
         const items = yield new __await(response.json());
         if (items.length === 0) {
@@ -1602,11 +1602,11 @@ var Client = class _Client {
       }
     });
   }
-  _getCursorPaginatedList(path2, body = null, requestMethod = "POST", dataKey = "runs") {
+  _getCursorPaginatedList(path, body = null, requestMethod = "POST", dataKey = "runs") {
     return __asyncGenerator(this, null, function* () {
       const bodyParams = body ? __spreadValues({}, body) : {};
       while (true) {
-        const response = yield new __await(this.caller.call(fetch, `${this.apiUrl}${path2}`, __spreadProps(__spreadValues({
+        const response = yield new __await(this.caller.call(fetch, `${this.apiUrl}${path}`, __spreadProps(__spreadValues({
           method: requestMethod,
           headers: __spreadProps(__spreadValues({}, this.headers), { "Content-Type": "application/json" }),
           signal: AbortSignal.timeout(this.timeout_ms)
@@ -2266,19 +2266,19 @@ var Client = class _Client {
     return result;
   }
   async hasProject({ projectId, projectName }) {
-    let path2 = "/sessions";
+    let path = "/sessions";
     const params = new URLSearchParams();
     if (projectId !== void 0 && projectName !== void 0) {
       throw new Error("Must provide either projectName or projectId, not both");
     } else if (projectId !== void 0) {
       assertUuid(projectId);
-      path2 += `/${projectId}`;
+      path += `/${projectId}`;
     } else if (projectName !== void 0) {
       params.append("name", projectName);
     } else {
       throw new Error("Must provide projectName or projectId");
     }
-    const response = await this.caller.call(fetch, `${this.apiUrl}${path2}?${params}`, __spreadValues({
+    const response = await this.caller.call(fetch, `${this.apiUrl}${path}?${params}`, __spreadValues({
       method: "GET",
       headers: this.headers,
       signal: AbortSignal.timeout(this.timeout_ms)
@@ -2297,13 +2297,13 @@ var Client = class _Client {
     }
   }
   async readProject({ projectId, projectName, includeStats }) {
-    let path2 = "/sessions";
+    let path = "/sessions";
     const params = new URLSearchParams();
     if (projectId !== void 0 && projectName !== void 0) {
       throw new Error("Must provide either projectName or projectId, not both");
     } else if (projectId !== void 0) {
       assertUuid(projectId);
-      path2 += `/${projectId}`;
+      path += `/${projectId}`;
     } else if (projectName !== void 0) {
       params.append("name", projectName);
     } else {
@@ -2312,7 +2312,7 @@ var Client = class _Client {
     if (includeStats !== void 0) {
       params.append("include_stats", includeStats.toString());
     }
-    const response = await this._get(path2, params);
+    const response = await this._get(path, params);
     let result;
     if (Array.isArray(response)) {
       if (response.length === 0) {
@@ -2484,19 +2484,19 @@ var Client = class _Client {
     return result;
   }
   async readDataset({ datasetId, datasetName }) {
-    let path2 = "/datasets";
+    let path = "/datasets";
     const params = new URLSearchParams({ limit: "1" });
     if (datasetId !== void 0 && datasetName !== void 0) {
       throw new Error("Must provide either datasetName or datasetId, not both");
     } else if (datasetId !== void 0) {
       assertUuid(datasetId);
-      path2 += `/${datasetId}`;
+      path += `/${datasetId}`;
     } else if (datasetName !== void 0) {
       params.append("name", datasetName);
     } else {
       throw new Error("Must provide datasetName or datasetId");
     }
-    const response = await this._get(path2, params);
+    const response = await this._get(path, params);
     let result;
     if (Array.isArray(response)) {
       if (response.length === 0) {
@@ -2540,21 +2540,21 @@ var Client = class _Client {
     return response;
   }
   async readDatasetOpenaiFinetuning({ datasetId, datasetName }) {
-    const path2 = "/datasets";
+    const path = "/datasets";
     if (datasetId !== void 0) {
     } else if (datasetName !== void 0) {
       datasetId = (await this.readDataset({ datasetName })).id;
     } else {
       throw new Error("Must provide datasetName or datasetId");
     }
-    const response = await this._getResponse(`${path2}/${datasetId}/openai_ft`);
+    const response = await this._getResponse(`${path}/${datasetId}/openai_ft`);
     const datasetText = await response.text();
     const dataset = datasetText.trim().split("\n").map((line) => JSON.parse(line));
     return dataset;
   }
   listDatasets() {
     return __asyncGenerator(this, arguments, function* ({ limit = 100, offset = 0, datasetIds, datasetName, datasetNameContains } = {}) {
-      const path2 = "/datasets";
+      const path = "/datasets";
       const params = new URLSearchParams({
         limit: limit.toString(),
         offset: offset.toString()
@@ -2571,7 +2571,7 @@ var Client = class _Client {
         params.append("name_contains", datasetNameContains);
       }
       try {
-        for (var iter = __forAwait(this._getPaginated(path2, params)), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
+        for (var iter = __forAwait(this._getPaginated(path, params)), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
           const datasets = temp.value;
           yield* __yieldStar(datasets);
         }
@@ -2611,7 +2611,7 @@ var Client = class _Client {
     return await response.json();
   }
   async deleteDataset({ datasetId, datasetName }) {
-    let path2 = "/datasets";
+    let path = "/datasets";
     let datasetId_ = datasetId;
     if (datasetId !== void 0 && datasetName !== void 0) {
       throw new Error("Must provide either datasetName or datasetId, not both");
@@ -2621,17 +2621,17 @@ var Client = class _Client {
     }
     if (datasetId_ !== void 0) {
       assertUuid(datasetId_);
-      path2 += `/${datasetId_}`;
+      path += `/${datasetId_}`;
     } else {
       throw new Error("Must provide datasetName or datasetId");
     }
-    const response = await this.caller.call(fetch, this.apiUrl + path2, __spreadValues({
+    const response = await this.caller.call(fetch, this.apiUrl + path, __spreadValues({
       method: "DELETE",
       headers: this.headers,
       signal: AbortSignal.timeout(this.timeout_ms)
     }, this.fetchOptions));
     if (!response.ok) {
-      throw new Error(`Failed to delete ${path2}: ${response.status} ${response.statusText}`);
+      throw new Error(`Failed to delete ${path}: ${response.status} ${response.statusText}`);
     }
     await response.json();
   }
@@ -2716,8 +2716,8 @@ var Client = class _Client {
   }
   async readExample(exampleId) {
     assertUuid(exampleId);
-    const path2 = `/examples/${exampleId}`;
-    return await this._get(path2);
+    const path = `/examples/${exampleId}`;
+    return await this._get(path);
   }
   listExamples() {
     return __asyncGenerator(this, arguments, function* ({ datasetId, datasetName, exampleIds, asOf, splits, inlineS3Urls, metadata, limit, offset, filter } = {}) {
@@ -2788,14 +2788,14 @@ var Client = class _Client {
   }
   async deleteExample(exampleId) {
     assertUuid(exampleId);
-    const path2 = `/examples/${exampleId}`;
-    const response = await this.caller.call(fetch, this.apiUrl + path2, __spreadValues({
+    const path = `/examples/${exampleId}`;
+    const response = await this.caller.call(fetch, this.apiUrl + path, __spreadValues({
       method: "DELETE",
       headers: this.headers,
       signal: AbortSignal.timeout(this.timeout_ms)
     }, this.fetchOptions));
     if (!response.ok) {
-      throw new Error(`Failed to delete ${path2}: ${response.status} ${response.statusText}`);
+      throw new Error(`Failed to delete ${path}: ${response.status} ${response.statusText}`);
     }
     await response.json();
   }
@@ -2899,20 +2899,20 @@ var Client = class _Client {
   }
   async readFeedback(feedbackId) {
     assertUuid(feedbackId);
-    const path2 = `/feedback/${feedbackId}`;
-    const response = await this._get(path2);
+    const path = `/feedback/${feedbackId}`;
+    const response = await this._get(path);
     return response;
   }
   async deleteFeedback(feedbackId) {
     assertUuid(feedbackId);
-    const path2 = `/feedback/${feedbackId}`;
-    const response = await this.caller.call(fetch, this.apiUrl + path2, __spreadValues({
+    const path = `/feedback/${feedbackId}`;
+    const response = await this.caller.call(fetch, this.apiUrl + path, __spreadValues({
       method: "DELETE",
       headers: this.headers,
       signal: AbortSignal.timeout(this.timeout_ms)
     }, this.fetchOptions));
     if (!response.ok) {
-      throw new Error(`Failed to delete ${path2}: ${response.status} ${response.statusText}`);
+      throw new Error(`Failed to delete ${path}: ${response.status} ${response.statusText}`);
     }
     await response.json();
   }
@@ -3898,12 +3898,12 @@ function traceable(wrappedFunc, config) {
     try {
       let runtimeConfig;
       if (argsConfigPath) {
-        const [index, path2] = argsConfigPath;
-        if (index === args.length - 1 && !path2) {
+        const [index, path] = argsConfigPath;
+        if (index === args.length - 1 && !path) {
           runtimeConfig = args.pop();
         } else if (index <= args.length && typeof args[index] === "object" && args[index] !== null) {
-          if (path2) {
-            const _a2 = args[index], { [path2]: extracted } = _a2, rest = __objRest(_a2, [__restKey(path2)]);
+          if (path) {
+            const _a2 = args[index], { [path]: extracted } = _a2, rest = __objRest(_a2, [__restKey(path)]);
             runtimeConfig = extracted;
             args[index] = rest;
           } else {
@@ -4244,6 +4244,7 @@ var import_openai = __toESM(require("openai"));
 var MIDSCENE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_OPENAI_INIT_CONFIG_JSON";
 var MIDSCENE_MODEL_NAME = "MIDSCENE_MODEL_NAME";
 var MIDSCENE_LANGSMITH_DEBUG = "MIDSCENE_LANGSMITH_DEBUG";
+var MIDSCENE_DEBUG_AI_PROFILE = "MIDSCENE_DEBUG_AI_PROFILE";
 var OPENAI_API_KEY = "OPENAI_API_KEY";
 function useOpenAIModel(useModel) {
   if (useModel && useModel !== "openAI")
@@ -4273,12 +4274,16 @@ async function createOpenAI() {
 }
 async function call(messages, responseFormat) {
   const openai = await createOpenAI();
+  const shouldPrintTiming = typeof process.env[MIDSCENE_DEBUG_AI_PROFILE] === "string";
+  shouldPrintTiming && console.time("Midscene - AI call");
   const completion = await openai.chat.completions.create({
     model,
     messages,
     response_format: { type: responseFormat },
     temperature: 0.2
   });
+  shouldPrintTiming && console.timeEnd("Midscene - AI call");
+  shouldPrintTiming && console.log("Midscene - AI usage", completion.usage);
   const { content } = completion.choices[0].message;
   (0, import_node_assert.default)(content, "empty content");
   return content;
@@ -4426,40 +4431,10 @@ function multiDescription(multi) {
 }
 
 // src/ai-model/prompt/util.ts
-var import_node_assert4 = __toESM(require("assert"));
-
-// src/image/info.ts
 var import_node_assert2 = __toESM(require("assert"));
-var import_node_buffer = require("buffer");
-var import_node_fs = require("fs");
-var import_sharp = __toESM(require("sharp"));
-async function imageInfo(image) {
-  const { width, height } = await (0, import_sharp.default)(image).metadata();
-  (0, import_node_assert2.default)(width && height, `invalid image: ${image}`);
-  return { width, height };
-}
-async function imageInfoOfBase64(imageBase64) {
-  const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
-  return imageInfo(import_node_buffer.Buffer.from(base64Data, "base64"));
-}
-
-// src/image/transform.ts
-var import_node_buffer2 = require("buffer");
-var import_sharp2 = __toESM(require("sharp"));
-
-// src/image/visualization.ts
-var import_node_buffer3 = require("buffer");
 
-// src/utils.ts
-var import_node_assert3 = __toESM(require("assert"));
-var import_node_crypto = require("crypto");
-var import_node_fs2 = require("fs");
-var import_node_os = require("os");
-var import_node_path = __toESM(require("path"));
-var logDir = (0, import_node_path.join)(process.cwd(), "./midscene_run/");
-
-// src/image/visualization.ts
-var import_sharp3 = __toESM(require("sharp"));
+// src/image/index.ts
+var import_img = require("@midscene/shared/img");
 
 // src/ai-model/prompt/util.ts
 var characteristic = "You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.";
@@ -4531,7 +4506,7 @@ async function describeUserPage(context) {
   if (context.size) {
     ({ width, height } = context.size);
   } else {
-    const imgSize = await imageInfoOfBase64(screenshotBase64);
+    const imgSize = await (0, import_img.imageInfoOfBase64)(screenshotBase64);
     ({ width, height } = imgSize);
   }
   const elementsInfo = context.content;
@@ -4552,7 +4527,7 @@ async function describeUserPage(context) {
       "elementInfos": ${JSON.stringify(elementInfosDescription)}
     }`,
     elementById(id) {
-      (0, import_node_assert4.default)(typeof id !== "undefined", "id is required for query");
+      (0, import_node_assert2.default)(typeof id !== "undefined", "id is required for query");
       const item = idElementMap[`${id}`];
       return item;
     }
@@ -4583,10 +4558,10 @@ function cropFieldInformation(elementsInfo) {
 }
 
 // src/ai-model/inspect.ts
-var import_node_assert6 = __toESM(require("assert"));
+var import_node_assert4 = __toESM(require("assert"));
 
 // src/ai-model/coze/index.ts
-var import_node_assert5 = __toESM(require("assert"));
+var import_node_assert3 = __toESM(require("assert"));
 var import_node_fetch = __toESM(require("node-fetch"));
 var COZE_INSPECT_ELEMENT_BOT_ID = process.env.COZE_INSPECT_ELEMENT_BOT_ID || "";
 var COZE_AI_ACTION_BOT_ID = process.env.COZE_AI_ACTION_BOT_ID || "";
@@ -4639,7 +4614,7 @@ async function callCozeAi(options) {
     throw new Error("aiResponse is undefined", aiResponse);
   }
   const parseContent = (_b = aiResponse == null ? void 0 : aiResponse.messages[0]) == null ? void 0 : _b.content;
-  (0, import_node_assert5.default)(parseContent, "empty content");
+  (0, import_node_assert3.default)(parseContent, "empty content");
   try {
     return JSON.parse(parseContent);
   } catch (err) {
@@ -4814,7 +4789,7 @@ DATA_DEMAND ends.
 }
 async function AiAssert(options) {
   const { assertion, context, useModel } = options;
-  (0, import_node_assert6.default)(assertion, "assertion should be a string");
+  (0, import_node_assert4.default)(assertion, "assertion should be a string");
   const { screenshotBase64 } = context;
   const { description, elementById } = await describeUserPage(context);
   const systemPrompt = systemPromptToAssert();
@@ -4858,7 +4833,7 @@ async function AiAssert(options) {
 }
 
 // src/ai-model/automation/index.ts
-var import_node_assert7 = __toESM(require("assert"));
+var import_node_assert5 = __toESM(require("assert"));
 
 // src/ai-model/automation/planning.ts
 function systemPromptToTaskPlanning() {
@@ -4964,8 +4939,8 @@ async function plan(userPrompt, opts, useModel) {
     });
   }
   const actions = (planFromAI == null ? void 0 : planFromAI.actions) || [];
-  (0, import_node_assert7.default)(planFromAI, "can't get planFromAI");
-  (0, import_node_assert7.default)(actions && actions.length > 0, "no actions in ai plan");
+  (0, import_node_assert5.default)(planFromAI, "can't get planFromAI");
+  (0, import_node_assert5.default)(actions && actions.length > 0, "no actions in ai plan");
   if (planFromAI.error) {
     throw new Error(planFromAI.error);
   }