@midscene/core 0.8.2-beta-20241115094249.0 → 0.8.3

package/dist/lib/types/index.d.ts

@@ -1,7 +1,7 @@
-import { E as ExecutionTask, a as ExecutionTaskApply, b as ExecutionDump, B as BaseElement, U as UIContext, D as DumpSubscriber, I as InsightOptions, c as InsightTaskInfo, A as AISingleElementResponse, d as InsightAssertionResponse } from './types-0d8eeece.js';
-export { l as AIAssertionResponse, h as AIElementIdResponse, i as AIElementPositionResponse, j as AIElementResponse, e as AIResponseFormat, k as AISectionParseResponse, f as AISingleElementResponseById, g as AISingleElementResponseByPosition, w as AgentAssertOpt, v as AgentWaitForOpt, Q as BaseAgentParserOpt, o as BasicSectionQuery, C as CallAIFn, O as Color, q as DumpMeta, u as ElementById, n as EnsureObject, W as ExecutionRecorderItem, ac as ExecutionTaskAction, ab as ExecutionTaskActionApply, aa as ExecutionTaskInsightAssertion, a9 as ExecutionTaskInsightAssertionApply, a8 as ExecutionTaskInsightAssertionParam, a1 as ExecutionTaskInsightDumpLog, a3 as ExecutionTaskInsightLocate, a2 as ExecutionTaskInsightLocateApply, a0 as ExecutionTaskInsightLocateOutput, $ as ExecutionTaskInsightLocateParam, a7 as ExecutionTaskInsightQuery, a6 as ExecutionTaskInsightQueryApply, a5 as ExecutionTaskInsightQueryOutput, a4 as ExecutionTaskInsightQueryParam, ae as ExecutionTaskPlanning, ad as ExecutionTaskPlanningApply, _ as ExecutionTaskReturn, X as ExecutionTaskType, Y as ExecutorContext, af as GroupedActionDump, s as InsightDump, p as InsightExtractParam, L as LiteUISection, t as PartialInsightDumpFromSDK, y as PlanningAIResponse, x as PlanningAction, J as PlanningActionParamAssert, M as PlanningActionParamError, F as PlanningActionParamHover, G as PlanningActionParamInputOrKeyPress, H as PlanningActionParamScroll, K as PlanningActionParamSleep, z as PlanningActionParamTap, N as PlanningActionParamWaitFor, V as PlaywrightParserOpt, P as Point, T as PuppeteerParserOpt, R as Rect, r as ReportDumpWithAttributes, S as Size, Z as TaskCacheInfo, m as UISection } from './types-0d8eeece.js';
-import { c as callAiFn, r as retrieveElement, a as retrieveSection } from './index-02a3ab02.js';
-export { b as allAIConfig, g as getAIConfig, o as overrideAIConfig, p as plan, t as transformElementPositionToId } from './index-02a3ab02.js';
+import { E as ExecutionTask, a as ExecutionTaskApply, b as ExecutionDump, B as BaseElement, U as UIContext, D as DumpSubscriber, I as InsightOptions, c as InsightTaskInfo, A as AISingleElementResponse, d as InsightAssertionResponse } from './types-29994b1b.js';
+export { l as AIAssertionResponse, h as AIElementIdResponse, i as AIElementPositionResponse, j as AIElementResponse, e as AIResponseFormat, k as AISectionParseResponse, f as AISingleElementResponseById, g as AISingleElementResponseByPosition, w as AgentAssertOpt, v as AgentWaitForOpt, Q as BaseAgentParserOpt, o as BasicSectionQuery, C as CallAIFn, O as Color, q as DumpMeta, u as ElementById, n as EnsureObject, W as ExecutionRecorderItem, ac as ExecutionTaskAction, ab as ExecutionTaskActionApply, aa as ExecutionTaskInsightAssertion, a9 as ExecutionTaskInsightAssertionApply, a8 as ExecutionTaskInsightAssertionParam, a1 as ExecutionTaskInsightDumpLog, a3 as ExecutionTaskInsightLocate, a2 as ExecutionTaskInsightLocateApply, a0 as ExecutionTaskInsightLocateOutput, $ as ExecutionTaskInsightLocateParam, a7 as ExecutionTaskInsightQuery, a6 as ExecutionTaskInsightQueryApply, a5 as ExecutionTaskInsightQueryOutput, a4 as ExecutionTaskInsightQueryParam, ae as ExecutionTaskPlanning, ad as ExecutionTaskPlanningApply, _ as ExecutionTaskReturn, X as ExecutionTaskType, Y as ExecutorContext, af as GroupedActionDump, s as InsightDump, p as InsightExtractParam, L as LiteUISection, t as PartialInsightDumpFromSDK, y as PlanningAIResponse, x as PlanningAction, J as PlanningActionParamAssert, M as PlanningActionParamError, F as PlanningActionParamHover, G as PlanningActionParamInputOrKeyPress, H as PlanningActionParamScroll, K as PlanningActionParamSleep, z as PlanningActionParamTap, N as PlanningActionParamWaitFor, V as PlaywrightParserOpt, P as Point, T as PuppeteerParserOpt, R as Rect, r as ReportDumpWithAttributes, S as Size, Z as TaskCacheInfo, m as UISection } from './types-29994b1b.js';
+import { c as callAiFn, r as retrieveElement, a as retrieveSection } from './index-690c2a06.js';
+export { b as allAIConfig, g as getAIConfig, o as overrideAIConfig, p as plan, t as transformElementPositionToId } from './index-690c2a06.js';
 export { getVersion, setLogDir } from './utils.js';
 import 'openai/resources';
 

package/dist/lib/types/{types-0d8eeece.d.ts → types-29994b1b.d.ts}

@@ -9,9 +9,7 @@ interface Size {
     height: number;
     dpr?: number;
 }
-type Rect = Point & Size & {
-    zoom?: number;
-};
+type Rect = Point & Size;
 declare enum NodeType {
     CONTAINER = "CONTAINER Node",
     FORM_ITEM = "FORM_ITEM Node",

package/dist/lib/types/utils.d.ts

@@ -1,4 +1,4 @@
-import { r as ReportDumpWithAttributes, R as Rect } from './types-0d8eeece.js';
+import { r as ReportDumpWithAttributes, R as Rect } from './types-29994b1b.js';
 import 'openai/resources';
 
 declare const insightDumpFileExt = "insight-dump.json";
@@ -22,5 +22,8 @@ declare function sleep(ms: number): Promise<unknown>;
 declare function replacerForPageObject(key: string, value: any): any;
 declare function stringifyDumpData(data: any, indents?: number): string;
 declare function getVersion(): string;
+declare function uploadTestInfoToServer({ testUrl, }: {
+    testUrl: string;
+}): void;
 
-export { getLogDir, getLogDirByType, getTmpDir, getTmpFile, getVersion, groupedActionDumpFileExt, insightDumpFileExt, overlapped, replacerForPageObject, reportHTMLContent, setLogDir, sleep, stringifyDumpData, writeDumpReport, writeLogFile };
+export { getLogDir, getLogDirByType, getTmpDir, getTmpFile, getVersion, groupedActionDumpFileExt, insightDumpFileExt, overlapped, replacerForPageObject, reportHTMLContent, setLogDir, sleep, stringifyDumpData, uploadTestInfoToServer, writeDumpReport, writeLogFile };

package/dist/lib/utils.js

@@ -43,16 +43,75 @@ __export(utils_exports, {
   setLogDir: () => setLogDir,
   sleep: () => sleep,
   stringifyDumpData: () => stringifyDumpData,
+  uploadTestInfoToServer: () => uploadTestInfoToServer,
   writeDumpReport: () => writeDumpReport,
   writeLogFile: () => writeLogFile
 });
 module.exports = __toCommonJS(utils_exports);
-var import_node_assert = __toESM(require("assert"));
+var import_node_assert4 = __toESM(require("assert"));
+var import_node_child_process = require("child_process");
 var import_node_fs = require("fs");
 var import_node_os = require("os");
 var import_node_path = require("path");
 var import_fs = require("@midscene/shared/fs");
+var import_utils2 = require("@midscene/shared/utils");
+
+// src/ai-model/openai/index.ts
+var import_node_assert3 = __toESM(require("assert"));
 var import_utils = require("@midscene/shared/utils");
+var import_openai5 = __toESM(require("openai"));
+
+// src/ai-model/coze/index.ts
+var import_node_assert = __toESM(require("assert"));
+var COZE_INSPECT_ELEMENT_BOT_ID = process.env.COZE_INSPECT_ELEMENT_BOT_ID || "";
+var COZE_AI_ACTION_BOT_ID = process.env.COZE_AI_ACTION_BOT_ID || "";
+var COZE_AI_ASSERT_BOT_ID = process.env.COZE_AI_ASSERT_BOT_ID || "";
+var COZE_EXTRACT_INFO_BOT_ID = process.env.COZE_EXTRACT_INFO_BOT_ID || "";
+
+// src/ai-model/prompt/util.ts
+var import_node_assert2 = __toESM(require("assert"));
+
+// src/image/index.ts
+var import_img = require("@midscene/shared/img");
+
+// src/ai-model/openai/index.ts
+var MIDSCENE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_OPENAI_INIT_CONFIG_JSON";
+var MIDSCENE_MODEL_NAME = "MIDSCENE_MODEL_NAME";
+var MIDSCENE_LANGSMITH_DEBUG = "MIDSCENE_LANGSMITH_DEBUG";
+var MIDSCENE_DEBUG_AI_PROFILE = "MIDSCENE_DEBUG_AI_PROFILE";
+var MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG = "MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG";
+var MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
+var OPENAI_API_KEY = "OPENAI_API_KEY";
+var OPENAI_BASE_URL = "OPENAI_BASE_URL";
+var MIDSCENE_MODEL_TEXT_ONLY = "MIDSCENE_MODEL_TEXT_ONLY";
+var OPENAI_USE_AZURE = "OPENAI_USE_AZURE";
+var MIDSCENE_CACHE = "MIDSCENE_CACHE";
+var MATCH_BY_POSITION = "MATCH_BY_POSITION";
+var allConfigFromEnv = () => {
+  return {
+    [MIDSCENE_OPENAI_INIT_CONFIG_JSON]: process.env[MIDSCENE_OPENAI_INIT_CONFIG_JSON] || void 0,
+    [MIDSCENE_MODEL_NAME]: process.env[MIDSCENE_MODEL_NAME] || void 0,
+    [MIDSCENE_DEBUG_MODE]: process.env[MIDSCENE_DEBUG_MODE] || void 0,
+    [MIDSCENE_LANGSMITH_DEBUG]: process.env[MIDSCENE_LANGSMITH_DEBUG] || void 0,
+    [MIDSCENE_DEBUG_AI_PROFILE]: process.env[MIDSCENE_DEBUG_AI_PROFILE] || void 0,
+    [MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG]: process.env[MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG] || void 0,
+    [OPENAI_API_KEY]: process.env[OPENAI_API_KEY] || void 0,
+    [OPENAI_BASE_URL]: process.env[OPENAI_BASE_URL] || void 0,
+    [MIDSCENE_MODEL_TEXT_ONLY]: process.env[MIDSCENE_MODEL_TEXT_ONLY] || void 0,
+    [OPENAI_USE_AZURE]: process.env[OPENAI_USE_AZURE] || void 0,
+    [MIDSCENE_CACHE]: process.env[MIDSCENE_CACHE] || void 0,
+    [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0
+  };
+};
+var userConfig = {};
+var getAIConfig = (configKey) => {
+  if (typeof userConfig[configKey] !== "undefined") {
+    return userConfig[configKey];
+  }
+  return allConfigFromEnv()[configKey];
+};
+
+// src/utils.ts
 var logDir = (0, import_node_path.join)(process.cwd(), "./midscene_run/");
 var logEnvReady = false;
 var insightDumpFileExt = "insight-dump.json";
@@ -72,11 +131,11 @@ function getLogDirByType(type) {
 }
 var reportTpl = null;
 function getReportTpl() {
-  if (import_utils.ifInBrowser) {
+  if (import_utils2.ifInBrowser) {
     if (!reportTpl && window.midscene_report_tpl) {
       reportTpl = window.midscene_report_tpl;
     }
-    (0, import_node_assert.default)(
+    (0, import_node_assert4.default)(
       reportTpl,
       "reportTpl should be set before writing report in browser"
     );
@@ -120,7 +179,7 @@ function reportHTMLContent(dumpData) {
   return reportContent;
 }
 function writeDumpReport(fileName, dumpData) {
-  if (import_utils.ifInBrowser) {
+  if (import_utils2.ifInBrowser) {
     console.log("will not write report in browser");
     return null;
   }
@@ -135,13 +194,13 @@ function writeDumpReport(fileName, dumpData) {
   return reportPath;
 }
 function writeLogFile(opts) {
-  if (import_utils.ifInBrowser) {
+  if (import_utils2.ifInBrowser) {
     return "/mock/report.html";
   }
   const { fileName, fileExt, fileContent, type = "dump" } = opts;
   const targetDir = getLogDirByType(type);
   if (!logEnvReady) {
-    (0, import_node_assert.default)(targetDir, "logDir should be set before writing dump file");
+    (0, import_node_assert4.default)(targetDir, "logDir should be set before writing dump file");
     const gitIgnorePath = (0, import_node_path.join)(targetDir, "../../.gitignore");
     let gitIgnoreContent = "";
     if ((0, import_node_fs.existsSync)(gitIgnorePath)) {
@@ -173,7 +232,7 @@ ${logDirName}/dump
   return filePath;
 }
 function getTmpDir() {
-  if (import_utils.ifInBrowser) {
+  if (import_utils2.ifInBrowser) {
     return null;
   }
   const runningPkgInfo = (0, import_fs.getRunningPkgInfo)();
@@ -186,11 +245,11 @@ function getTmpDir() {
   return path;
 }
 function getTmpFile(fileExtWithoutDot) {
-  if (import_utils.ifInBrowser) {
+  if (import_utils2.ifInBrowser) {
     return null;
   }
   const tmpDir = getTmpDir();
-  const filename = `${(0, import_utils.uuid)()}.${fileExtWithoutDot}`;
+  const filename = `${(0, import_utils2.uuid)()}.${fileExtWithoutDot}`;
   return (0, import_node_path.join)(tmpDir, filename);
 }
 function overlapped(container, target) {
@@ -213,7 +272,49 @@ function stringifyDumpData(data, indents) {
   return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-  return "0.8.2-beta-20241115094249.0";
+  return "0.8.3";
+}
+function debugLog(...message) {
+  const debugMode = getAIConfig(MIDSCENE_DEBUG_MODE);
+  if (debugMode) {
+    console.log("[Midscene]", ...message);
+  }
+}
+var lastReportedRepoUrl = "";
+function uploadTestInfoToServer({
+  testUrl
+}) {
+  let repoUrl = "";
+  const extraConfigString = getAIConfig(MIDSCENE_OPENAI_INIT_CONFIG_JSON);
+  const extraConfig = extraConfigString ? JSON.parse(extraConfigString) : {};
+  const serverUrl = extraConfig == null ? void 0 : extraConfig.REPORT_SERVER_URL;
+  try {
+    repoUrl = (0, import_node_child_process.execSync)("git config --get remote.origin.url").toString().trim();
+  } catch (error) {
+    debugLog("Failed to get git repo URL:", error);
+  }
+  if (serverUrl && (repoUrl && repoUrl !== lastReportedRepoUrl || !repoUrl && testUrl)) {
+    debugLog("Uploading test info to server", {
+      serverUrl,
+      repoUrl,
+      testUrl
+    });
+    fetch(serverUrl, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify({
+        repo_url: repoUrl,
+        test_url: testUrl
+      })
+    }).then((response) => response.json()).then((data) => {
+      debugLog("Successfully uploaded test info to server:", data);
+    }).catch(
+      (error) => debugLog("Failed to upload test info to server:", error)
+    );
+    lastReportedRepoUrl = repoUrl;
+  }
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
@@ -230,6 +331,7 @@ function getVersion() {
   setLogDir,
   sleep,
   stringifyDumpData,
+  uploadTestInfoToServer,
   writeDumpReport,
   writeLogFile
 });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "An AI-powered automation SDK can control the page, perform assertions, and extract data in JSON format using natural language. See https://midscenejs.com/ for details.",
-  "version": "0.8.2-beta-20241115094249.0",
+  "version": "0.8.3",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "jsnext:source": "./src/index.ts",
@@ -34,7 +34,7 @@
   "dependencies": {
     "openai": "4.57.1",
     "optional": "0.1.4",
-    "@midscene/shared": "0.8.2-beta-20241115094249.0"
+    "@midscene/shared": "0.8.3"
   },
   "devDependencies": {
     "@modern-js/module-tools": "2.60.6",