@midscene/core 0.0.1

package/dist/lib/utils.js

@@ -0,0 +1,141 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/utils.ts
+var utils_exports = {};
+__export(utils_exports, {
+  commonScreenshotParam: () => commonScreenshotParam,
+  getDumpDir: () => getDumpDir,
+  getPkgInfo: () => getPkgInfo,
+  getTmpDir: () => getTmpDir,
+  getTmpFile: () => getTmpFile,
+  groupedActionDumpFileExt: () => groupedActionDumpFileExt,
+  insightDumpFileExt: () => insightDumpFileExt,
+  overlapped: () => overlapped,
+  setDumpDir: () => setDumpDir,
+  sleep: () => sleep,
+  writeDumpFile: () => writeDumpFile
+});
+module.exports = __toCommonJS(utils_exports);
+var import_os = require("os");
+var import_path = require("path");
+var import_fs = require("fs");
+var import_crypto = require("crypto");
+var import_assert = __toESM(require("assert"));
+var pkg;
+function getPkgInfo() {
+  if (pkg) {
+    return pkg;
+  }
+  let pkgJsonFile = "";
+  if ((0, import_fs.existsSync)((0, import_path.join)(__dirname, "../package.json"))) {
+    pkgJsonFile = (0, import_path.join)(__dirname, "../package.json");
+  } else if ((0, import_fs.existsSync)((0, import_path.join)(__dirname, "../../../package.json"))) {
+    pkgJsonFile = (0, import_path.join)(__dirname, "../../../package.json");
+  }
+  if (pkgJsonFile) {
+    const { name, version } = JSON.parse((0, import_fs.readFileSync)(pkgJsonFile, "utf-8"));
+    pkg = { name, version };
+    return pkg;
+  } else {
+    return {
+      name: "midscene-unknown-page-name",
+      version: "0.0.0"
+    };
+  }
+}
+var logDir = (0, import_path.join)(process.cwd(), "./midscene_run/");
+var logEnvReady = false;
+var insightDumpFileExt = "insight-dump.json";
+var groupedActionDumpFileExt = "web-dump.json";
+function getDumpDir() {
+  return logDir;
+}
+function setDumpDir(dir) {
+  logDir = dir;
+}
+function writeDumpFile(fileName, fileExt, fileContent) {
+  if (!logEnvReady) {
+    (0, import_assert.default)(logDir, "logDir should be set before writing dump file");
+    if (!(0, import_fs.existsSync)(logDir)) {
+      (0, import_fs.mkdirSync)(logDir, { recursive: true });
+    }
+    const gitIgnorePath = (0, import_path.join)(logDir, "../.gitignore");
+    let gitIgnoreContent = "";
+    if ((0, import_fs.existsSync)(gitIgnorePath)) {
+      gitIgnoreContent = (0, import_fs.readFileSync)(gitIgnorePath, "utf-8");
+    }
+    const logDirName = (0, import_path.basename)(logDir);
+    if (!gitIgnoreContent.includes(`${logDirName}/`)) {
+      (0, import_fs.writeFileSync)(
+        gitIgnorePath,
+        `${gitIgnoreContent}
+# MidScene.js dump files
+${logDirName}/
+`,
+        "utf-8"
+      );
+    }
+    logEnvReady = true;
+  }
+  const filePath = (0, import_path.join)(getDumpDir(), `${fileName}.${fileExt}`);
+  (0, import_fs.writeFileSync)(filePath, fileContent);
+  (0, import_fs.copyFileSync)(filePath, (0, import_path.join)(getDumpDir(), `latest.${fileExt}`));
+  return filePath;
+}
+function getTmpDir() {
+  const path = (0, import_path.join)((0, import_os.tmpdir)(), getPkgInfo().name);
+  (0, import_fs.mkdirSync)(path, { recursive: true });
+  return path;
+}
+function getTmpFile(fileExt) {
+  const filename = `${(0, import_crypto.randomUUID)()}.${fileExt}`;
+  return (0, import_path.join)(getTmpDir(), filename);
+}
+function overlapped(container, target) {
+  return container.left < target.left + target.width && container.left + container.width > target.left && container.top < target.top + target.height && container.top + container.height > target.top;
+}
+async function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+var commonScreenshotParam = { type: "jpeg", quality: 75 };
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  commonScreenshotParam,
+  getDumpDir,
+  getPkgInfo,
+  getTmpDir,
+  getTmpFile,
+  groupedActionDumpFileExt,
+  insightDumpFileExt,
+  overlapped,
+  setDumpDir,
+  sleep,
+  writeDumpFile
+});

package/dist/types/ai-model.d.ts

@@ -0,0 +1,32 @@
+import { c as callToGetJSONObject } from './util-3a13ce3d.js';
+export { d as describeUserPage } from './util-3a13ce3d.js';
+export { ChatCompletionMessageParam } from 'openai/resources';
+import { B as BaseElement, U as UIContext, e as AIElementParseResponse, f as AISectionParseResponse } from './types-1f7912d5.js';
+
+declare function systemPromptToFindElement(description: string, multi?: boolean): string;
+
+declare function AiInspectElement<ElementType extends BaseElement = BaseElement>(options: {
+    context: UIContext<ElementType>;
+    multi: boolean;
+    findElementDescription: string;
+    callAI?: typeof callToGetJSONObject;
+}): Promise<{
+    parseResult: AIElementParseResponse;
+    elementById: (id: string) => ElementType;
+    systemPrompt: string;
+}>;
+declare function AiExtractElementInfo<T, ElementType extends BaseElement = BaseElement>(options: {
+    dataQuery: string | Record<string, string>;
+    sectionConstraints: {
+        name: string;
+        description: string;
+    }[];
+    context: UIContext<ElementType>;
+    callAI?: typeof callToGetJSONObject;
+}): Promise<{
+    parseResult: AISectionParseResponse<T>;
+    elementById: (id: string) => ElementType;
+    systemPrompt: string;
+}>;
+
+export { AiExtractElementInfo, AiInspectElement, callToGetJSONObject, systemPromptToFindElement };

package/dist/types/image.d.ts

@@ -0,0 +1,119 @@
+import { Buffer } from 'buffer';
+import { S as Size, R as Rect, g as UISection, U as UIContext, t as Color } from './types-1f7912d5.js';
+import { Buffer as Buffer$1 } from 'node:buffer';
+import 'openai/resources';
+
+/**
+ * Retrieves the dimensions of an image asynchronously
+ *
+ * @param image - The image data, which can be a string path or a buffer
+ * @returns A Promise that resolves to an object containing the width and height of the image
+ * @throws Error if the image data is invalid
+ */
+declare function imageInfo(image: string | Buffer): Promise<Size>;
+/**
+ * Retrieves the dimensions of an image from a base64-encoded string
+ *
+ * @param imageBase64 - The base64-encoded image data
+ * @returns A Promise that resolves to an object containing the width and height of the image
+ * @throws Error if the image data is invalid
+ */
+declare function imageInfoOfBase64(imageBase64: string): Promise<Size>;
+/**
+ * Encodes an image file to a base64 encoded string
+ *
+ * @param image The path of the image file
+ * @param withHeader Determine whether to return data including the file header information, the default is true
+ *
+ * @returns The base64 encoded string of the image file, which may or may not include header information depending on the withHeader parameter
+ *
+ * @throws When the image type is not supported, an error will be thrown
+ */
+declare function base64Encoded(image: string, withHeader?: boolean): string;
+
+/**
+ * Saves a Base64-encoded image to a file
+ *
+ * @param options - An object containing the Base64-encoded image data and the output file path
+ * @param options.base64Data - The Base64-encoded image data
+ * @param options.outputPath - The path where the image will be saved
+ * @throws Error if there is an error during the saving process
+ */
+declare function saveBase64Image(options: {
+    base64Data: string;
+    outputPath: string;
+}): Promise<void>;
+/**
+ * Transforms an image path into a base64-encoded string
+ * @param inputPath - The path of the image file to be encoded
+ * @returns A Promise that resolves to a base64-encoded string representing the image file
+ */
+declare function transformImgPathToBase64(inputPath: string): Promise<string>;
+/**
+ * Resizes an image from a base64-encoded string
+ *
+ * @param base64Data - A base64-encoded string representing the image
+ * @returns A Promise that resolves to a base64-encoded string representing the resized image
+ * @throws An error if the width or height cannot be determined from the metadata
+ */
+declare function resizeImg(base64Data: string): Promise<string>;
+/**
+ * Calculates new dimensions for an image while maintaining its aspect ratio.
+ *
+ * This function is designed to resize an image to fit within a specified maximum width and height
+ * while maintaining the original aspect ratio. If the original width or height exceeds the maximum
+ * dimensions, the image will be scaled down to fit.
+ *
+ * @param {number} originalWidth - The original width of the image.
+ * @param {number} originalHeight - The original height of the image.
+ * @returns {Object} An object containing the new width and height.
+ * @throws {Error} Throws an error if the width or height is not a positive number.
+ */
+declare function calculateNewDimensions(originalWidth: number, originalHeight: number): {
+    width: number;
+    height: number;
+};
+/**
+ * Trims an image and returns the trimming information, including the offset from the left and top edges, and the trimmed width and height
+ *
+ * @param image - The image to be trimmed. This can be a file path or a Buffer object containing the image data
+ * @returns A Promise that resolves to an object containing the trimming information. If the image does not need to be trimmed, this object will be null
+ */
+declare function trimImage(image: string | Buffer$1): Promise<{
+    trimOffsetLeft: number;
+    trimOffsetTop: number;
+    width: number;
+    height: number;
+} | null>;
+/**
+ * Aligns an image's coordinate system based on trimming information
+ *
+ * This function takes an image and a center rectangle as input. It first extracts the center
+ * rectangle from the image using the Sharp library and converts it to a buffer. Then, it calls
+ * the trimImage function to obtain the trimming information of the buffer image. If there is no
+ * trimming information, the original center rectangle is returned. If there is trimming information,
+ * a new rectangle is created based on the trimming information, with its top-left corner
+ * positioned at the negative offset of the trimming from the original center rectangle's top-left
+ * corner, and its width and height set to the trimmed image's dimensions.
+ *
+ * @param image The image file path or buffer to be processed
+ * @param center The center rectangle of the image, which is used to extract and align
+ * @returns A Promise that resolves to a rectangle object representing the aligned coordinates
+ * @throws Error if there is an error during image processing
+ */
+declare function alignCoordByTrim(image: string | Buffer$1, center: Rect): Promise<Rect>;
+
+/**
+ * Composes a section diagram based on the given sections and context
+ * It creates an SVG representation of the sections and converts it to a PNG image file
+ *
+ * @param sections - An array of UISection objects representing the sections to be included in the diagram
+ * @param context - The UIContext object containing the size information for the diagram
+ * @returns {Promise<{ file: string; sectionNameColorMap: Record<string, Color>; }>}
+ */
+declare function composeSectionDiagram(sections: UISection[], context: UIContext): Promise<{
+    file: string;
+    sectionNameColorMap: Record<string, Color>;
+}>;
+
+export { alignCoordByTrim, base64Encoded, calculateNewDimensions, composeSectionDiagram, imageInfo, imageInfoOfBase64, resizeImg, saveBase64Image, transformImgPathToBase64, trimImage };

package/dist/types/index.d.ts

@@ -0,0 +1,43 @@
+import { c as callToGetJSONObject, r as retrieveElement, a as retrieveSection } from './util-3a13ce3d.js';
+import { B as BaseElement, U as UIContext, D as DumpSubscriber, I as InsightTaskInfo, a as InsightOptions, E as ExecutionTask, b as ExecutionTaskApply, c as ExecutionDump, P as PlanningAction } from './types-1f7912d5.js';
+export { e as AIElementParseResponse, A as AIResponseFormat, f as AISectionParseResponse, u as BaseAgentParserOpt, i as BasicSectionQuery, C as CallAIFn, t as Color, k as DumpMeta, n as ElementById, h as EnsureObject, X as ExectuionTaskPlanningParam, x as ExecutionRecorderItem, W as ExecutionTaskAction, V as ExecutionTaskActionApply, M as ExecutionTaskInsightLocate, K as ExecutionTaskInsightLocateApply, J as ExecutionTaskInsightLocateLog, H as ExecutionTaskInsightLocateOutput, G as ExecutionTaskInsightLocateParam, T as ExecutionTaskInsightQuery, Q as ExecutionTaskInsightQueryApply, O as ExecutionTaskInsightQueryOutput, N as ExecutionTaskInsightQueryParam, Z as ExecutionTaskPlanning, Y as ExecutionTaskPlanningApply, F as ExecutionTaskReturn, y as ExecutionTaskType, z as ExecutorContext, _ as GroupedActionDump, l as InsightDump, j as InsightExtractParam, L as LiteUISection, m as PartialInsightDumpFromSDK, o as PlanningAIResponse, q as PlanningActionParamHover, r as PlanningActionParamInputOrKeyPress, s as PlanningActionParamScroll, p as PlanningActionParamTap, w as PlaywrightParserOpt, d as Point, v as PuppeteerParserOpt, R as Rect, S as Size, g as UISection } from './types-1f7912d5.js';
+export { setDumpDir } from './utils.js';
+import 'openai/resources';
+
+declare class Insight<ElementType extends BaseElement = BaseElement> {
+    contextRetrieverFn: () => Promise<UIContext<ElementType>> | UIContext<ElementType>;
+    aiVendorFn: typeof callToGetJSONObject;
+    onceDumpUpdatedFn?: DumpSubscriber;
+    taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
+    constructor(context: UIContext<ElementType> | (() => Promise<UIContext<ElementType>> | UIContext<ElementType>), opt?: InsightOptions);
+    locate(queryPrompt: string): Promise<ElementType | null>;
+    locate(queryPrompt: string, opt: {
+        multi: true;
+    }): Promise<ElementType[]>;
+    extract<T = any>(input: string): Promise<T>;
+    extract<T extends Record<string, string>>(input: T): Promise<Record<keyof T, any>>;
+    extract<T extends object>(input: Record<keyof T, string>): Promise<T>;
+}
+
+declare class Executor {
+    name: string;
+    description?: string;
+    tasks: ExecutionTask[];
+    status: 'init' | 'pending' | 'running' | 'completed' | 'error';
+    errorMsg?: string;
+    dumpFileName?: string;
+    constructor(name: string, description?: string, tasks?: ExecutionTaskApply[]);
+    private markTaskAsPending;
+    append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void>;
+    flush(): Promise<void>;
+    dump(): ExecutionDump;
+}
+
+declare const getElement: typeof retrieveElement;
+declare const getSection: typeof retrieveSection;
+
+declare function plan(context: UIContext, userPrompt: string): Promise<{
+    plans: PlanningAction[];
+}>;
+
+export { BaseElement, DumpSubscriber, ExecutionDump, ExecutionTask, ExecutionTaskApply, Executor, InsightOptions, InsightTaskInfo, PlanningAction, UIContext, Insight as default, getElement, getSection, plan };

package/dist/types/types-1f7912d5.d.ts

@@ -0,0 +1,219 @@
+import { ChatCompletionMessageParam } from 'openai/resources';
+
+interface Point {
+    left: number;
+    top: number;
+}
+interface Size {
+    width: number;
+    height: number;
+}
+type Rect = Point & Size;
+declare enum NodeType {
+    'INPUT' = "INPUT Node",
+    'BUTTON' = "BUTTON Node",
+    'IMG' = "IMG Node",
+    'TEXT' = "TEXT Node"
+}
+declare abstract class BaseElement {
+    abstract id: string;
+    abstract attributes: {
+        ['nodeType']: NodeType;
+        [key: string]: string;
+    };
+    abstract content: string;
+    abstract rect: Rect;
+    abstract center: [number, number];
+    abstract locator?: string;
+}
+/**
+ * openai
+ *
+ */
+declare enum AIResponseFormat {
+    JSON = "json_object",
+    TEXT = "text"
+}
+interface AIElementParseResponse {
+    elements: {
+        id: string;
+        reason: string;
+        text: string;
+    }[];
+    errors?: string[];
+}
+interface AISectionParseResponse<DataShape> {
+    data: DataShape;
+    sections?: LiteUISection[];
+    errors?: string[];
+}
+/**
+ * context
+ */
+declare abstract class UIContext<ElementType extends BaseElement = BaseElement> {
+    abstract screenshotBase64: string;
+    abstract content: ElementType[];
+    abstract size: Size;
+}
+/**
+ * insight
+ */
+type CallAIFn = <T>(messages: ChatCompletionMessageParam[]) => Promise<T>;
+interface InsightOptions {
+    taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
+    aiVendorFn?: CallAIFn;
+}
+interface UISection {
+    name: string;
+    description: string;
+    sectionCharacteristics: string;
+    rect: Rect;
+    content: BaseElement[];
+}
+type EnsureObject<T> = {
+    [K in keyof T]: any;
+};
+interface BasicSectionQuery {
+    name?: string;
+    description?: string;
+}
+type InsightExtractParam = string | Record<string, string>;
+interface InsightTaskInfo {
+    durationMs: number;
+    systemPrompt?: string;
+    rawResponse?: string;
+}
+interface DumpMeta {
+    sdkVersion: string;
+    logTime: number;
+}
+interface InsightDump extends DumpMeta {
+    type: 'locate' | 'extract';
+    logId: string;
+    context: UIContext;
+    userQuery: {
+        element?: string;
+        dataDemand?: InsightExtractParam;
+        sections?: Record<string, string>;
+    };
+    matchedSection: UISection[];
+    matchedElement: BaseElement[];
+    data: any;
+    taskInfo: InsightTaskInfo;
+    error?: string;
+}
+type PartialInsightDumpFromSDK = Omit<InsightDump, 'sdkVersion' | 'logTime' | 'logId'>;
+type DumpSubscriber = (dump: InsightDump) => Promise<void> | void;
+interface LiteUISection {
+    name: string;
+    description: string;
+    sectionCharacteristics: string;
+    textIds: string[];
+}
+type ElementById = (id: string) => BaseElement | null;
+/**
+ * planning
+ *
+ */
+interface PlanningAction<ParamType = any> {
+    thought: string;
+    type: 'Locate' | 'Tap' | 'Hover' | 'Input' | 'KeyboardPress' | 'Scroll' | 'Error';
+    param: ParamType;
+}
+interface PlanningAIResponse {
+    queryLanguage: string;
+    actions: PlanningAction[];
+    error?: string;
+}
+type PlanningActionParamTap = null;
+type PlanningActionParamHover = null;
+interface PlanningActionParamInputOrKeyPress {
+    value: string;
+}
+interface PlanningActionParamScroll {
+    scrollType: 'ScrollUntilBottom' | 'ScrollUntilTop' | 'ScrollDown' | 'ScrollUp';
+}
+/**
+ * misc
+ */
+interface Color {
+    name: string;
+    hex: string;
+}
+interface BaseAgentParserOpt {
+    selector: string;
+}
+interface PuppeteerParserOpt extends BaseAgentParserOpt {
+}
+interface PlaywrightParserOpt extends BaseAgentParserOpt {
+}
+interface ExecutionRecorderItem {
+    type: 'screenshot';
+    ts: number;
+    screenshot?: string;
+    timing?: string;
+}
+type ExecutionTaskType = 'Planning' | 'Insight' | 'Action' | 'Assertion';
+interface ExecutorContext {
+    task: ExecutionTask;
+    element?: BaseElement | null;
+}
+interface ExecutionTaskApply<Type extends ExecutionTaskType = any, TaskParam = any, TaskOutput = any, TaskLog = any> {
+    type: Type;
+    subType?: string;
+    param?: TaskParam;
+    executor: (param: TaskParam, context: ExecutorContext) => Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | void> | void;
+}
+interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {
+    output?: TaskOutput;
+    log?: TaskLog;
+    recorder?: ExecutionRecorderItem[];
+}
+type ExecutionTask<E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<any, any, any>> = E & ExecutionTaskReturn<E extends ExecutionTaskApply<any, any, infer TaskOutput, any> ? TaskOutput : unknown, E extends ExecutionTaskApply<any, any, any, infer TaskLog> ? TaskLog : unknown> & {
+    status: 'pending' | 'running' | 'success' | 'fail' | 'cancelled';
+    error?: string;
+    timing?: {
+        start: number;
+        end?: number;
+        cost?: number;
+    };
+};
+interface ExecutionDump extends DumpMeta {
+    name: string;
+    description?: string;
+    tasks: ExecutionTask[];
+}
+interface ExecutionTaskInsightLocateParam {
+    prompt: string;
+}
+interface ExecutionTaskInsightLocateOutput {
+    element: BaseElement | null;
+}
+interface ExecutionTaskInsightLocateLog {
+    dump?: InsightDump;
+}
+type ExecutionTaskInsightLocateApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightLocateParam, ExecutionTaskInsightLocateOutput, ExecutionTaskInsightLocateLog>;
+type ExecutionTaskInsightLocate = ExecutionTask<ExecutionTaskInsightLocateApply>;
+interface ExecutionTaskInsightQueryParam {
+    dataDemand: InsightExtractParam;
+}
+interface ExecutionTaskInsightQueryOutput {
+    data: any;
+}
+type ExecutionTaskInsightQueryApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightQueryParam>;
+type ExecutionTaskInsightQuery = ExecutionTask<ExecutionTaskInsightQueryApply>;
+type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<'Action', ActionParam, void, void>;
+type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;
+type ExectuionTaskPlanningParam = PlanningAIResponse;
+type ExecutionTaskPlanningApply = ExecutionTaskApply<'Planning', {
+    userPrompt: string;
+}, {
+    plans: PlanningAction[];
+}>;
+type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;
+interface GroupedActionDump {
+    groupName: string;
+    executions: ExecutionDump[];
+}
+
+export { AIResponseFormat as A, BaseElement as B, type CallAIFn as C, type DumpSubscriber as D, type ExecutionTask as E, type ExecutionTaskReturn as F, type ExecutionTaskInsightLocateParam as G, type ExecutionTaskInsightLocateOutput as H, type InsightTaskInfo as I, type ExecutionTaskInsightLocateLog as J, type ExecutionTaskInsightLocateApply as K, type LiteUISection as L, type ExecutionTaskInsightLocate as M, type ExecutionTaskInsightQueryParam as N, type ExecutionTaskInsightQueryOutput as O, type PlanningAction as P, type ExecutionTaskInsightQueryApply as Q, type Rect as R, type Size as S, type ExecutionTaskInsightQuery as T, UIContext as U, type ExecutionTaskActionApply as V, type ExecutionTaskAction as W, type ExectuionTaskPlanningParam as X, type ExecutionTaskPlanningApply as Y, type ExecutionTaskPlanning as Z, type GroupedActionDump as _, type InsightOptions as a, type ExecutionTaskApply as b, type ExecutionDump as c, type Point as d, type AIElementParseResponse as e, type AISectionParseResponse as f, type UISection as g, type EnsureObject as h, type BasicSectionQuery as i, type InsightExtractParam as j, type DumpMeta as k, type InsightDump as l, type PartialInsightDumpFromSDK as m, type ElementById as n, type PlanningAIResponse as o, type PlanningActionParamTap as p, type PlanningActionParamHover as q, type PlanningActionParamInputOrKeyPress as r, type PlanningActionParamScroll as s, type Color as t, type BaseAgentParserOpt as u, type PuppeteerParserOpt as v, type PlaywrightParserOpt as w, type ExecutionRecorderItem as x, type ExecutionTaskType as y, type ExecutorContext as z };

package/dist/types/util-3a13ce3d.d.ts

@@ -0,0 +1,21 @@
+import { ChatCompletionMessageParam } from 'openai/resources';
+import { B as BaseElement, U as UIContext } from './types-1f7912d5.js';
+
+declare function callToGetJSONObject<T>(messages: ChatCompletionMessageParam[]): Promise<T>;
+
+declare function describeUserPage<ElementType extends BaseElement = BaseElement>(context: Omit<UIContext<ElementType>, 'describer'>): Promise<{
+    description: string;
+    elementById(id: string): ElementType;
+}>;
+/**
+ * elements
+ */
+declare function retrieveElement(prompt: string, opt?: {
+    multi: boolean;
+}): string;
+/**
+ * sections
+ */
+declare function retrieveSection(prompt: string): string;
+
+export { retrieveSection as a, callToGetJSONObject as c, describeUserPage as d, retrieveElement as r };

package/dist/types/utils.d.ts

@@ -0,0 +1,20 @@
+import { R as Rect } from './types-1f7912d5.js';
+import 'openai/resources';
+
+interface PkgInfo {
+    name: string;
+    version: string;
+}
+declare function getPkgInfo(): PkgInfo;
+declare const insightDumpFileExt = "insight-dump.json";
+declare const groupedActionDumpFileExt = "web-dump.json";
+declare function getDumpDir(): string;
+declare function setDumpDir(dir: string): void;
+declare function writeDumpFile(fileName: string, fileExt: string, fileContent: string): string;
+declare function getTmpDir(): string;
+declare function getTmpFile(fileExt: string): string;
+declare function overlapped(container: Rect, target: Rect): boolean;
+declare function sleep(ms: number): Promise<unknown>;
+declare const commonScreenshotParam: any;
+
+export { commonScreenshotParam, getDumpDir, getPkgInfo, getTmpDir, getTmpFile, groupedActionDumpFileExt, insightDumpFileExt, overlapped, setDumpDir, sleep, writeDumpFile };

package/modern.config.ts

@@ -0,0 +1,18 @@
+import { moduleTools, defineConfig } from '@modern-js/module-tools';
+
+export default defineConfig({
+  plugins: [moduleTools()],
+  buildPreset: 'npm-library',
+  buildConfig: {
+    platform: 'node',
+    input: {
+      index: 'src/index.ts',
+      utils: 'src/utils.ts',
+      image: 'src/image/index.ts',
+      'ai-model': 'src/ai-model/index.ts',
+    },
+    // input: ['src/utils.ts', 'src/index.ts', 'src/image/index.ts'],
+    externals: ['langsmith/wrappers', 'buffer'],
+    target: 'es2017'
+  },
+});