@midscene/core 0.0.1 → 0.1.0

package/dist/es/ai-model.js

@@ -314,7 +314,14 @@ function truncateText(text) {
 }
 async function describeUserPage(context) {
   const { screenshotBase64 } = context;
-  const { width, height } = await imageInfoOfBase64(screenshotBase64);
+  let width;
+  let height;
+  if (context.size) {
+    ({ width, height } = context.size);
+  } else {
+    const imgSize = await imageInfoOfBase64(screenshotBase64);
+    ({ width, height } = imgSize);
+  }
   const elementsInfo = context.content;
   const idElementMap = {};
   elementsInfo.forEach((item) => {

package/dist/es/image.js

@@ -140,8 +140,8 @@ function getTmpDir() {
   mkdirSync(path, { recursive: true });
   return path;
 }
-function getTmpFile(fileExt) {
-  const filename = `${randomUUID()}.${fileExt}`;
+function getTmpFile(fileExtWithoutDot) {
+  const filename = `${randomUUID()}.${fileExtWithoutDot}`;
   return join(getTmpDir(), filename);
 }
 

package/dist/es/index.js

@@ -100,13 +100,18 @@ function getDumpDir() {
 function setDumpDir(dir) {
   logDir = dir;
 }
-function writeDumpFile(fileName, fileExt, fileContent) {
+function getDumpDirPath(type) {
+  return join(getDumpDir(), type);
+}
+function writeDumpFile(opts) {
+  const { fileName, fileExt, fileContent, type = "dump" } = opts;
+  const targetDir = getDumpDirPath(type);
+  if (!existsSync(targetDir)) {
+    mkdirSync(targetDir, { recursive: true });
+  }
   if (!logEnvReady) {
-    assert2(logDir, "logDir should be set before writing dump file");
-    if (!existsSync(logDir)) {
-      mkdirSync(logDir, { recursive: true });
-    }
-    const gitIgnorePath = join(logDir, "../.gitignore");
+    assert2(targetDir, "logDir should be set before writing dump file");
+    const gitIgnorePath = join(targetDir, "../../.gitignore");
     let gitIgnoreContent = "";
     if (existsSync(gitIgnorePath)) {
       gitIgnoreContent = readFileSync2(gitIgnorePath, "utf-8");
@@ -117,16 +122,19 @@ function writeDumpFile(fileName, fileExt, fileContent) {
         gitIgnorePath,
         `${gitIgnoreContent}
 # MidScene.js dump files
-${logDirName}/
+${logDirName}/midscene-report
+${logDirName}/dump-logger
 `,
         "utf-8"
       );
     }
     logEnvReady = true;
   }
-  const filePath = join(getDumpDir(), `${fileName}.${fileExt}`);
+  const filePath = join(targetDir, `${fileName}.${fileExt}`);
   writeFileSync(filePath, fileContent);
-  copyFileSync(filePath, join(getDumpDir(), `latest.${fileExt}`));
+  if (type === "dump") {
+    copyFileSync(filePath, join(targetDir, `latest.${fileExt}`));
+  }
   return filePath;
 }
 
@@ -222,7 +230,14 @@ function truncateText(text) {
 }
 async function describeUserPage(context) {
   const { screenshotBase64 } = context;
-  const { width, height } = await imageInfoOfBase64(screenshotBase64);
+  let width;
+  let height;
+  if (context.size) {
+    ({ width, height } = context.size);
+  } else {
+    const imgSize = await imageInfoOfBase64(screenshotBase64);
+    ({ width, height } = imgSize);
+  }
   const elementsInfo = context.content;
   const idElementMap = {};
   elementsInfo.forEach((item) => {
@@ -341,9 +356,13 @@ function writeInsightDump(data, logId, dumpSubscriber) {
     const length = logContent.push(dataString);
     logIdIndexMap[id] = length - 1;
   }
-  writeDumpFile(logFileName, logFileExt, `[
+  writeDumpFile({
+    fileName: logFileName,
+    fileExt: logFileExt,
+    fileContent: `[
 ${logContent.join(",\n")}
-]`);
+]`
+  });
   return id;
 }
 function idsIntoElements(ids, elementById) {
@@ -675,15 +694,16 @@ var Insight = class {
   }
   async locate(queryPrompt, opt) {
     var _a;
+    const { callAI = this.aiVendorFn, multi = false } = opt || {};
     assert6(queryPrompt, "query is required for located");
     const dumpSubscriber = this.onceDumpUpdatedFn;
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn();
     const startTime = Date.now();
     const { parseResult, systemPrompt, elementById } = await AiInspectElement({
-      callAI: this.aiVendorFn,
+      callAI,
       context,
-      multi: Boolean(opt == null ? void 0 : opt.multi),
+      multi: Boolean(multi),
       findElementDescription: queryPrompt
     });
     const timeCost = Date.now() - startTime;
@@ -850,6 +870,13 @@ ${parseResult.errors.join("\n")}`;
     );
     return mergedData;
   }
+  setAiVendorFn(aiVendorFn) {
+    const origin = this.aiVendorFn;
+    this.aiVendorFn = aiVendorFn;
+    return () => {
+      this.aiVendorFn = origin;
+    };
+  }
 };
 
 // src/action/executor.ts
@@ -1032,7 +1059,8 @@ function systemPromptToTaskPlanning(query) {
   =====================================
   `;
 }
-async function plan(context, userPrompt) {
+async function plan(userPrompt, opts) {
+  const { callAI = callToGetJSONObject, context } = opts || {};
   const { screenshotBase64 } = context;
   const { description } = await describeUserPage(context);
   const systemPrompt = systemPromptToTaskPlanning(userPrompt);
@@ -1055,7 +1083,7 @@ async function plan(context, userPrompt) {
       ]
     }
   ];
-  const planFromAI = await callToGetJSONObject(msgs);
+  const planFromAI = await callAI(msgs);
   if (planFromAI.error) {
     throw new Error(planFromAI.error);
   }

package/dist/es/utils.js

@@ -36,13 +36,18 @@ function getDumpDir() {
 function setDumpDir(dir) {
   logDir = dir;
 }
-function writeDumpFile(fileName, fileExt, fileContent) {
+function getDumpDirPath(type) {
+  return join(getDumpDir(), type);
+}
+function writeDumpFile(opts) {
+  const { fileName, fileExt, fileContent, type = "dump" } = opts;
+  const targetDir = getDumpDirPath(type);
+  if (!existsSync(targetDir)) {
+    mkdirSync(targetDir, { recursive: true });
+  }
   if (!logEnvReady) {
-    assert(logDir, "logDir should be set before writing dump file");
-    if (!existsSync(logDir)) {
-      mkdirSync(logDir, { recursive: true });
-    }
-    const gitIgnorePath = join(logDir, "../.gitignore");
+    assert(targetDir, "logDir should be set before writing dump file");
+    const gitIgnorePath = join(targetDir, "../../.gitignore");
     let gitIgnoreContent = "";
     if (existsSync(gitIgnorePath)) {
       gitIgnoreContent = readFileSync(gitIgnorePath, "utf-8");
@@ -53,16 +58,19 @@ function writeDumpFile(fileName, fileExt, fileContent) {
         gitIgnorePath,
         `${gitIgnoreContent}
 # MidScene.js dump files
-${logDirName}/
+${logDirName}/midscene-report
+${logDirName}/dump-logger
 `,
         "utf-8"
       );
     }
     logEnvReady = true;
   }
-  const filePath = join(getDumpDir(), `${fileName}.${fileExt}`);
+  const filePath = join(targetDir, `${fileName}.${fileExt}`);
   writeFileSync(filePath, fileContent);
-  copyFileSync(filePath, join(getDumpDir(), `latest.${fileExt}`));
+  if (type === "dump") {
+    copyFileSync(filePath, join(targetDir, `latest.${fileExt}`));
+  }
   return filePath;
 }
 function getTmpDir() {
@@ -70,8 +78,8 @@ function getTmpDir() {
   mkdirSync(path, { recursive: true });
   return path;
 }
-function getTmpFile(fileExt) {
-  const filename = `${randomUUID()}.${fileExt}`;
+function getTmpFile(fileExtWithoutDot) {
+  const filename = `${randomUUID()}.${fileExtWithoutDot}`;
   return join(getTmpDir(), filename);
 }
 function overlapped(container, target) {
@@ -84,6 +92,7 @@ var commonScreenshotParam = { type: "jpeg", quality: 75 };
 export {
   commonScreenshotParam,
   getDumpDir,
+  getDumpDirPath,
   getPkgInfo,
   getTmpDir,
   getTmpFile,

package/dist/lib/ai-model.js

@@ -351,7 +351,14 @@ function truncateText(text) {
 }
 async function describeUserPage(context) {
   const { screenshotBase64 } = context;
-  const { width, height } = await imageInfoOfBase64(screenshotBase64);
+  let width;
+  let height;
+  if (context.size) {
+    ({ width, height } = context.size);
+  } else {
+    const imgSize = await imageInfoOfBase64(screenshotBase64);
+    ({ width, height } = imgSize);
+  }
   const elementsInfo = context.content;
   const idElementMap = {};
   elementsInfo.forEach((item) => {

package/dist/lib/image.js

@@ -185,8 +185,8 @@ function getTmpDir() {
   (0, import_fs.mkdirSync)(path, { recursive: true });
   return path;
 }
-function getTmpFile(fileExt) {
-  const filename = `${(0, import_crypto.randomUUID)()}.${fileExt}`;
+function getTmpFile(fileExtWithoutDot) {
+  const filename = `${(0, import_crypto.randomUUID)()}.${fileExtWithoutDot}`;
   return (0, import_path.join)(getTmpDir(), filename);
 }
 

package/dist/lib/index.js

@@ -141,13 +141,18 @@ function getDumpDir() {
 function setDumpDir(dir) {
   logDir = dir;
 }
-function writeDumpFile(fileName, fileExt, fileContent) {
+function getDumpDirPath(type) {
+  return (0, import_path.join)(getDumpDir(), type);
+}
+function writeDumpFile(opts) {
+  const { fileName, fileExt, fileContent, type = "dump" } = opts;
+  const targetDir = getDumpDirPath(type);
+  if (!(0, import_fs.existsSync)(targetDir)) {
+    (0, import_fs.mkdirSync)(targetDir, { recursive: true });
+  }
   if (!logEnvReady) {
-    (0, import_assert.default)(logDir, "logDir should be set before writing dump file");
-    if (!(0, import_fs.existsSync)(logDir)) {
-      (0, import_fs.mkdirSync)(logDir, { recursive: true });
-    }
-    const gitIgnorePath = (0, import_path.join)(logDir, "../.gitignore");
+    (0, import_assert.default)(targetDir, "logDir should be set before writing dump file");
+    const gitIgnorePath = (0, import_path.join)(targetDir, "../../.gitignore");
     let gitIgnoreContent = "";
     if ((0, import_fs.existsSync)(gitIgnorePath)) {
       gitIgnoreContent = (0, import_fs.readFileSync)(gitIgnorePath, "utf-8");
@@ -158,16 +163,19 @@ function writeDumpFile(fileName, fileExt, fileContent) {
         gitIgnorePath,
         `${gitIgnoreContent}
 # MidScene.js dump files
-${logDirName}/
+${logDirName}/midscene-report
+${logDirName}/dump-logger
 `,
         "utf-8"
       );
     }
     logEnvReady = true;
   }
-  const filePath = (0, import_path.join)(getDumpDir(), `${fileName}.${fileExt}`);
+  const filePath = (0, import_path.join)(targetDir, `${fileName}.${fileExt}`);
   (0, import_fs.writeFileSync)(filePath, fileContent);
-  (0, import_fs.copyFileSync)(filePath, (0, import_path.join)(getDumpDir(), `latest.${fileExt}`));
+  if (type === "dump") {
+    (0, import_fs.copyFileSync)(filePath, (0, import_path.join)(targetDir, `latest.${fileExt}`));
+  }
   return filePath;
 }
 
@@ -263,7 +271,14 @@ function truncateText(text) {
 }
 async function describeUserPage(context) {
   const { screenshotBase64 } = context;
-  const { width, height } = await imageInfoOfBase64(screenshotBase64);
+  let width;
+  let height;
+  if (context.size) {
+    ({ width, height } = context.size);
+  } else {
+    const imgSize = await imageInfoOfBase64(screenshotBase64);
+    ({ width, height } = imgSize);
+  }
   const elementsInfo = context.content;
   const idElementMap = {};
   elementsInfo.forEach((item) => {
@@ -382,9 +397,13 @@ function writeInsightDump(data, logId, dumpSubscriber) {
     const length = logContent.push(dataString);
     logIdIndexMap[id] = length - 1;
   }
-  writeDumpFile(logFileName, logFileExt, `[
+  writeDumpFile({
+    fileName: logFileName,
+    fileExt: logFileExt,
+    fileContent: `[
 ${logContent.join(",\n")}
-]`);
+]`
+  });
   return id;
 }
 function idsIntoElements(ids, elementById) {
@@ -716,15 +735,16 @@ var Insight = class {
   }
   async locate(queryPrompt, opt) {
     var _a;
+    const { callAI = this.aiVendorFn, multi = false } = opt || {};
     (0, import_assert5.default)(queryPrompt, "query is required for located");
     const dumpSubscriber = this.onceDumpUpdatedFn;
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn();
     const startTime = Date.now();
     const { parseResult, systemPrompt, elementById } = await AiInspectElement({
-      callAI: this.aiVendorFn,
+      callAI,
       context,
-      multi: Boolean(opt == null ? void 0 : opt.multi),
+      multi: Boolean(multi),
       findElementDescription: queryPrompt
     });
     const timeCost = Date.now() - startTime;
@@ -891,6 +911,13 @@ ${parseResult.errors.join("\n")}`;
     );
     return mergedData;
   }
+  setAiVendorFn(aiVendorFn) {
+    const origin = this.aiVendorFn;
+    this.aiVendorFn = aiVendorFn;
+    return () => {
+      this.aiVendorFn = origin;
+    };
+  }
 };
 
 // src/action/executor.ts
@@ -1073,7 +1100,8 @@ function systemPromptToTaskPlanning(query) {
   =====================================
   `;
 }
-async function plan(context, userPrompt) {
+async function plan(userPrompt, opts) {
+  const { callAI = callToGetJSONObject, context } = opts || {};
   const { screenshotBase64 } = context;
   const { description } = await describeUserPage(context);
   const systemPrompt = systemPromptToTaskPlanning(userPrompt);
@@ -1096,7 +1124,7 @@ async function plan(context, userPrompt) {
       ]
     }
   ];
-  const planFromAI = await callToGetJSONObject(msgs);
+  const planFromAI = await callAI(msgs);
   if (planFromAI.error) {
     throw new Error(planFromAI.error);
   }

package/dist/lib/utils.js

@@ -32,6 +32,7 @@ var utils_exports = {};
 __export(utils_exports, {
   commonScreenshotParam: () => commonScreenshotParam,
   getDumpDir: () => getDumpDir,
+  getDumpDirPath: () => getDumpDirPath,
   getPkgInfo: () => getPkgInfo,
   getTmpDir: () => getTmpDir,
   getTmpFile: () => getTmpFile,
@@ -80,13 +81,18 @@ function getDumpDir() {
 function setDumpDir(dir) {
   logDir = dir;
 }
-function writeDumpFile(fileName, fileExt, fileContent) {
+function getDumpDirPath(type) {
+  return (0, import_path.join)(getDumpDir(), type);
+}
+function writeDumpFile(opts) {
+  const { fileName, fileExt, fileContent, type = "dump" } = opts;
+  const targetDir = getDumpDirPath(type);
+  if (!(0, import_fs.existsSync)(targetDir)) {
+    (0, import_fs.mkdirSync)(targetDir, { recursive: true });
+  }
   if (!logEnvReady) {
-    (0, import_assert.default)(logDir, "logDir should be set before writing dump file");
-    if (!(0, import_fs.existsSync)(logDir)) {
-      (0, import_fs.mkdirSync)(logDir, { recursive: true });
-    }
-    const gitIgnorePath = (0, import_path.join)(logDir, "../.gitignore");
+    (0, import_assert.default)(targetDir, "logDir should be set before writing dump file");
+    const gitIgnorePath = (0, import_path.join)(targetDir, "../../.gitignore");
     let gitIgnoreContent = "";
     if ((0, import_fs.existsSync)(gitIgnorePath)) {
       gitIgnoreContent = (0, import_fs.readFileSync)(gitIgnorePath, "utf-8");
@@ -97,16 +103,19 @@ function writeDumpFile(fileName, fileExt, fileContent) {
         gitIgnorePath,
         `${gitIgnoreContent}
 # MidScene.js dump files
-${logDirName}/
+${logDirName}/midscene-report
+${logDirName}/dump-logger
 `,
         "utf-8"
       );
     }
     logEnvReady = true;
   }
-  const filePath = (0, import_path.join)(getDumpDir(), `${fileName}.${fileExt}`);
+  const filePath = (0, import_path.join)(targetDir, `${fileName}.${fileExt}`);
   (0, import_fs.writeFileSync)(filePath, fileContent);
-  (0, import_fs.copyFileSync)(filePath, (0, import_path.join)(getDumpDir(), `latest.${fileExt}`));
+  if (type === "dump") {
+    (0, import_fs.copyFileSync)(filePath, (0, import_path.join)(targetDir, `latest.${fileExt}`));
+  }
   return filePath;
 }
 function getTmpDir() {
@@ -114,8 +123,8 @@ function getTmpDir() {
   (0, import_fs.mkdirSync)(path, { recursive: true });
   return path;
 }
-function getTmpFile(fileExt) {
-  const filename = `${(0, import_crypto.randomUUID)()}.${fileExt}`;
+function getTmpFile(fileExtWithoutDot) {
+  const filename = `${(0, import_crypto.randomUUID)()}.${fileExtWithoutDot}`;
   return (0, import_path.join)(getTmpDir(), filename);
 }
 function overlapped(container, target) {
@@ -129,6 +138,7 @@ var commonScreenshotParam = { type: "jpeg", quality: 75 };
 0 && (module.exports = {
   commonScreenshotParam,
   getDumpDir,
+  getDumpDirPath,
   getPkgInfo,
   getTmpDir,
   getTmpFile,

package/dist/types/ai-model.d.ts

@@ -9,7 +9,7 @@ declare function AiInspectElement<ElementType extends BaseElement = BaseElement>
     context: UIContext<ElementType>;
     multi: boolean;
     findElementDescription: string;
-    callAI?: typeof callToGetJSONObject;
+    callAI?: typeof callToGetJSONObject<AIElementParseResponse>;
 }): Promise<{
     parseResult: AIElementParseResponse;
     elementById: (id: string) => ElementType;

package/dist/types/index.d.ts

@@ -4,19 +4,26 @@ export { e as AIElementParseResponse, A as AIResponseFormat, f as AISectionParse
 export { setDumpDir } from './utils.js';
 import 'openai/resources';
 
-declare class Insight<ElementType extends BaseElement = BaseElement> {
-    contextRetrieverFn: () => Promise<UIContext<ElementType>> | UIContext<ElementType>;
+interface LocateOpts {
+    multi?: boolean;
+    callAI?: typeof callToGetJSONObject<AIElementParseResponse>;
+}
+declare class Insight<ElementType extends BaseElement = BaseElement, ContextType extends UIContext<ElementType> = UIContext<ElementType>> {
+    contextRetrieverFn: () => Promise<ContextType> | ContextType;
     aiVendorFn: typeof callToGetJSONObject;
     onceDumpUpdatedFn?: DumpSubscriber;
     taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
-    constructor(context: UIContext<ElementType> | (() => Promise<UIContext<ElementType>> | UIContext<ElementType>), opt?: InsightOptions);
-    locate(queryPrompt: string): Promise<ElementType | null>;
+    constructor(context: ContextType | (() => Promise<ContextType> | ContextType), opt?: InsightOptions);
+    locate(queryPrompt: string, opt?: {
+        callAI: LocateOpts['callAI'];
+    }): Promise<ElementType | null>;
     locate(queryPrompt: string, opt: {
         multi: true;
     }): Promise<ElementType[]>;
     extract<T = any>(input: string): Promise<T>;
     extract<T extends Record<string, string>>(input: T): Promise<Record<keyof T, any>>;
     extract<T extends object>(input: Record<keyof T, string>): Promise<T>;
+    setAiVendorFn<T>(aiVendorFn: typeof callToGetJSONObject<T>): () => void;
 }
 
 declare class Executor {
@@ -36,7 +43,10 @@ declare class Executor {
 declare const getElement: typeof retrieveElement;
 declare const getSection: typeof retrieveSection;
 
-declare function plan(context: UIContext, userPrompt: string): Promise<{
+declare function plan(userPrompt: string, opts: {
+    context: UIContext;
+    callAI?: typeof callToGetJSONObject<PlanningAIResponse>;
+}): Promise<{
     plans: PlanningAction[];
 }>;
 

package/dist/types/utils.d.ts

@@ -10,11 +10,17 @@ declare const insightDumpFileExt = "insight-dump.json";
 declare const groupedActionDumpFileExt = "web-dump.json";
 declare function getDumpDir(): string;
 declare function setDumpDir(dir: string): void;
-declare function writeDumpFile(fileName: string, fileExt: string, fileContent: string): string;
+declare function getDumpDirPath(type: 'dump' | 'cache'): string;
+declare function writeDumpFile(opts: {
+    fileName: string;
+    fileExt: string;
+    fileContent: string;
+    type?: 'dump' | 'cache';
+}): string;
 declare function getTmpDir(): string;
-declare function getTmpFile(fileExt: string): string;
+declare function getTmpFile(fileExtWithoutDot: string): string;
 declare function overlapped(container: Rect, target: Rect): boolean;
 declare function sleep(ms: number): Promise<unknown>;
 declare const commonScreenshotParam: any;
 
-export { commonScreenshotParam, getDumpDir, getPkgInfo, getTmpDir, getTmpFile, groupedActionDumpFileExt, insightDumpFileExt, overlapped, setDumpDir, sleep, writeDumpFile };
+export { commonScreenshotParam, getDumpDir, getDumpDirPath, getPkgInfo, getTmpDir, getTmpFile, groupedActionDumpFileExt, insightDumpFileExt, overlapped, setDumpDir, sleep, writeDumpFile };

package/package.json

@@ -1,11 +1,15 @@
 {
   "name": "@midscene/core",
   "description": "Hello, It's MidScene",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "jsnext:source": "./src/index.ts",
   "main": "./dist/lib/index.js",
   "module": "./dist/es/index.js",
   "types": "./dist/types/index.d.ts",
+  "files": [
+    "dist",
+    "README.md"
+  ],
   "exports": {
     ".": {
       "types": "./dist/types/index.d.ts",

package/.eslintignore

@@ -1,3 +0,0 @@
-demo_data
-vitest.config.ts
-modern.config.ts

package/.eslintrc.js

@@ -1,9 +0,0 @@
-module.exports = {
-  root: true,
-  rules: {
-    '@typescript-eslint/no-explicit-any': 'warn',
-    '@typescript-eslint/no-magic-numbers': 'off',
-    '@typescript-eslint/no-parameter-properties': 'off',
-  },
-  extends: ['@modern-js'],
-};

package/CONTRIBUTING.md

@@ -1,5 +0,0 @@
-It's all driven by tests:
-
-```shell
-npx vitest ./tests/
-```