@donggui/core 1.5.4-donggui.3

package/dist/es/screenshot-item.mjs

@@ -0,0 +1,105 @@
+import { readFileSync } from "node:fs";
+import { uuid } from "@midscene/shared/utils";
+import { extractImageByIdSync } from "./dump/html-utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function detectFormat(base64) {
+    if (base64.startsWith('data:image/jpeg')) return 'jpeg';
+    if (base64.startsWith('data:image/jpg')) return 'jpeg';
+    return 'png';
+}
+class ScreenshotItem {
+    static create(base64, capturedAt) {
+        return new ScreenshotItem(uuid(), base64, capturedAt);
+    }
+    get id() {
+        return this._id;
+    }
+    get format() {
+        return this._format;
+    }
+    get extension() {
+        return 'jpeg' === this._format ? 'jpeg' : 'png';
+    }
+    get capturedAt() {
+        return this._capturedAt;
+    }
+    get base64() {
+        if (null !== this._base64) return this._base64;
+        if (null !== this._persistedPath) {
+            const buffer = readFileSync(this._persistedPath);
+            return `data:image/${this._format};base64,${buffer.toString('base64')}`;
+        }
+        if (null !== this._persistedHtmlPath) {
+            const data = extractImageByIdSync(this._persistedHtmlPath, this._id);
+            if (data) return data;
+            throw new Error(`Screenshot ${this._id}: cannot recover from HTML (id not found in ${this._persistedHtmlPath})`);
+        }
+        throw new Error(`Screenshot ${this._id}: base64 data released without recovery path`);
+    }
+    hasBase64() {
+        return null !== this._base64;
+    }
+    markPersistedInline(htmlPath) {
+        this._persistedAs = {
+            $screenshot: this._id,
+            capturedAt: this._capturedAt
+        };
+        this._persistedHtmlPath = htmlPath;
+        this._base64 = null;
+    }
+    markPersistedToPath(relativePath, absolutePath) {
+        this._persistedAs = {
+            base64: relativePath,
+            capturedAt: this._capturedAt
+        };
+        this._persistedPath = absolutePath;
+        this._base64 = null;
+    }
+    toSerializable() {
+        return this._persistedAs ?? {
+            $screenshot: this._id,
+            capturedAt: this._capturedAt
+        };
+    }
+    static isSerialized(value) {
+        if ('object' != typeof value || null === value) return false;
+        const record = value;
+        if ('$screenshot' in record && 'string' == typeof record.$screenshot) {
+            if (!('capturedAt' in record) || 'number' != typeof record.capturedAt) return false;
+            return true;
+        }
+        if ('base64' in record && 'string' == typeof record.base64) {
+            if (!('capturedAt' in record) || 'number' != typeof record.capturedAt) return false;
+            return true;
+        }
+        return false;
+    }
+    get rawBase64() {
+        return this.base64.replace(/^data:image\/(png|jpeg|jpg);base64,/, '');
+    }
+    constructor(id, base64, capturedAt){
+        _define_property(this, "_id", void 0);
+        _define_property(this, "_base64", void 0);
+        _define_property(this, "_format", void 0);
+        _define_property(this, "_capturedAt", void 0);
+        _define_property(this, "_persistedAs", null);
+        _define_property(this, "_persistedPath", null);
+        _define_property(this, "_persistedHtmlPath", null);
+        this._id = id;
+        this._base64 = base64;
+        this._format = detectFormat(base64);
+        this._capturedAt = capturedAt;
+    }
+}
+export { ScreenshotItem };
+
+//# sourceMappingURL=screenshot-item.mjs.map

package/dist/es/screenshot-item.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"screenshot-item.mjs","sources":["../../src/screenshot-item.ts"],"sourcesContent":["import { readFileSync } from 'node:fs';\nimport { uuid } from '@midscene/shared/utils';\nimport { extractImageByIdSync } from './dump/html-utils';\n\n/**\n * Serialization format for ScreenshotItem\n * - { $screenshot: \"id\" } - inline mode, references imageMap in HTML\n * - { base64: \"path\" } - directory mode, references external file path\n */\nexport type ScreenshotSerializeFormat =\n  | { $screenshot: string; capturedAt: number }\n  | { base64: string; capturedAt: number };\n\n/**\n * Detect image format from base64 data URI prefix.\n */\nfunction detectFormat(base64: string): 'png' | 'jpeg' {\n  if (base64.startsWith('data:image/jpeg')) return 'jpeg';\n  if (base64.startsWith('data:image/jpg')) return 'jpeg';\n  return 'png';\n}\n\n/**\n * ScreenshotItem encapsulates screenshot data.\n *\n * Supports lazy loading after memory release:\n * - inline mode: reads from HTML file using streaming (extractImageByIdSync)\n * - directory mode: reads from file on disk\n *\n * After persistence, memory is released but the screenshot can be recovered\n * on-demand from disk, making it safe to release memory at any time.\n */\nexport class ScreenshotItem {\n  private _id: string;\n  private _base64: string | null;\n  private _format: 'png' | 'jpeg';\n  private _capturedAt: number;\n  private _persistedAs: ScreenshotSerializeFormat | null = null;\n  private _persistedPath: string | null = null;\n  private _persistedHtmlPath: string | null = null;\n\n  private constructor(id: string, base64: string, capturedAt: number) {\n    this._id = id;\n    this._base64 = base64;\n    this._format = detectFormat(base64);\n    this._capturedAt = capturedAt;\n  }\n\n  /** Create a new ScreenshotItem from base64 data */\n  static create(base64: string, capturedAt: number): ScreenshotItem {\n    return new ScreenshotItem(uuid(), base64, capturedAt);\n  }\n\n  get id(): string {\n    return this._id;\n  }\n\n  /** Get the image format (png or jpeg) */\n  get format(): 'png' | 'jpeg' {\n    return this._format;\n  }\n\n  /** Get the file extension for this screenshot */\n  get extension(): string {\n    return this._format === 'jpeg' ? 'jpeg' : 'png';\n  }\n\n  /** Get screenshot capture timestamp in milliseconds */\n  get capturedAt(): number {\n    return this._capturedAt;\n  }\n\n  get base64(): string {\n    // If data is in memory, return it directly\n    if (this._base64 !== null) {\n      return this._base64;\n    }\n\n    // Directory mode: recover from file\n    if (this._persistedPath !== null) {\n      const buffer = readFileSync(this._persistedPath);\n      return `data:image/${this._format};base64,${buffer.toString('base64')}`;\n    }\n\n    // Inline mode: recover from HTML file using streaming\n    if (this._persistedHtmlPath !== null) {\n      const data = extractImageByIdSync(this._persistedHtmlPath, this._id);\n      if (data) {\n        return data;\n      }\n      throw new Error(\n        `Screenshot ${this._id}: cannot recover from HTML (id not found in ${this._persistedHtmlPath})`,\n      );\n    }\n\n    throw new Error(\n      `Screenshot ${this._id}: base64 data released without recovery path`,\n    );\n  }\n\n  /** Check if base64 data is still available in memory (not yet released) */\n  hasBase64(): boolean {\n    return this._base64 !== null;\n  }\n\n  /**\n   * Mark as persisted to HTML (inline mode).\n   * Releases base64 memory, but keeps HTML path for lazy loading recovery.\n   * @param htmlPath - absolute path to the HTML file containing the image\n   */\n  markPersistedInline(htmlPath: string): void {\n    this._persistedAs = {\n      $screenshot: this._id,\n      capturedAt: this._capturedAt,\n    };\n    this._persistedHtmlPath = htmlPath;\n    this._base64 = null;\n  }\n\n  /**\n   * Mark as persisted to file (directory mode).\n   * Releases base64 memory, but keeps file path for lazy loading recovery.\n   * @param relativePath - relative path for serialization (e.g., \"./screenshots/id.jpeg\")\n   * @param absolutePath - absolute path for lazy loading recovery\n   */\n  markPersistedToPath(relativePath: string, absolutePath: string): void {\n    this._persistedAs = {\n      base64: relativePath,\n      capturedAt: this._capturedAt,\n    };\n    this._persistedPath = absolutePath;\n    this._base64 = null;\n  }\n\n  /** Serialize for JSON - format depends on persistence state */\n  toSerializable(): ScreenshotSerializeFormat {\n    return (\n      this._persistedAs ?? {\n        $screenshot: this._id,\n        capturedAt: this._capturedAt,\n      }\n    );\n  }\n\n  /** Check if a value is a serialized ScreenshotItem reference (inline or directory mode) */\n  static isSerialized(value: unknown): value is ScreenshotSerializeFormat {\n    if (typeof value !== 'object' || value === null) return false;\n    const record = value as Record<string, unknown>;\n    // Check for inline mode: { $screenshot: string }\n    if ('$screenshot' in record && typeof record.$screenshot === 'string') {\n      if (!('capturedAt' in record) || typeof record.capturedAt !== 'number') {\n        return false;\n      }\n      return true;\n    }\n    // Check for directory mode: { base64: string } where base64 is a path\n    if ('base64' in record && typeof record.base64 === 'string') {\n      if (!('capturedAt' in record) || typeof record.capturedAt !== 'number') {\n        return false;\n      }\n      return true;\n    }\n    return false;\n  }\n\n  /**\n   * Get base64 data without the data URI prefix.\n   * Useful for writing raw binary data to files.\n   */\n  get rawBase64(): string {\n    return this.base64.replace(/^data:image\\/(png|jpeg|jpg);base64,/, '');\n  }\n}\n"],"names":["detectFormat","base64","ScreenshotItem","capturedAt","uuid","buffer","readFileSync","data","extractImageByIdSync","Error","htmlPath","relativePath","absolutePath","value","record","id"],"mappings":";;;;;;;;;;;;;AAgBA,SAASA,aAAaC,MAAc;IAClC,IAAIA,OAAO,UAAU,CAAC,oBAAoB,OAAO;IACjD,IAAIA,OAAO,UAAU,CAAC,mBAAmB,OAAO;IAChD,OAAO;AACT;AAYO,MAAMC;IAiBX,OAAO,OAAOD,MAAc,EAAEE,UAAkB,EAAkB;QAChE,OAAO,IAAID,eAAeE,QAAQH,QAAQE;IAC5C;IAEA,IAAI,KAAa;QACf,OAAO,IAAI,CAAC,GAAG;IACjB;IAGA,IAAI,SAAyB;QAC3B,OAAO,IAAI,CAAC,OAAO;IACrB;IAGA,IAAI,YAAoB;QACtB,OAAO,AAAiB,WAAjB,IAAI,CAAC,OAAO,GAAc,SAAS;IAC5C;IAGA,IAAI,aAAqB;QACvB,OAAO,IAAI,CAAC,WAAW;IACzB;IAEA,IAAI,SAAiB;QAEnB,IAAI,AAAiB,SAAjB,IAAI,CAAC,OAAO,EACd,OAAO,IAAI,CAAC,OAAO;QAIrB,IAAI,AAAwB,SAAxB,IAAI,CAAC,cAAc,EAAW;YAChC,MAAME,SAASC,aAAa,IAAI,CAAC,cAAc;YAC/C,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAED,OAAO,QAAQ,CAAC,WAAW;QACzE;QAGA,IAAI,AAA4B,SAA5B,IAAI,CAAC,kBAAkB,EAAW;YACpC,MAAME,OAAOC,qBAAqB,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,GAAG;YACnE,IAAID,MACF,OAAOA;YAET,MAAM,IAAIE,MACR,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,4CAA4C,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAEnG;QAEA,MAAM,IAAIA,MACR,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,4CAA4C,CAAC;IAExE;IAGA,YAAqB;QACnB,OAAO,AAAiB,SAAjB,IAAI,CAAC,OAAO;IACrB;IAOA,oBAAoBC,QAAgB,EAAQ;QAC1C,IAAI,CAAC,YAAY,GAAG;YAClB,aAAa,IAAI,CAAC,GAAG;YACrB,YAAY,IAAI,CAAC,WAAW;QAC9B;QACA,IAAI,CAAC,kBAAkB,GAAGA;QAC1B,IAAI,CAAC,OAAO,GAAG;IACjB;IAQA,oBAAoBC,YAAoB,EAAEC,YAAoB,EAAQ;QACpE,IAAI,CAAC,YAAY,GAAG;YAClB,QAAQD;YACR,YAAY,IAAI,CAAC,WAAW;QAC9B;QACA,IAAI,CAAC,cAAc,GAAGC;QACtB,IAAI,CAAC,OAAO,GAAG;IACjB;IAGA,iBAA4C;QAC1C,OACE,IAAI,CAAC,YAAY,IAAI;YACnB,aAAa,IAAI,CAAC,GAAG;YACrB,YAAY,IAAI,CAAC,WAAW;QAC9B;IAEJ;IAGA,OAAO,aAAaC,KAAc,EAAsC;QACtE,IAAI,AAAiB,YAAjB,OAAOA,SAAsBA,AAAU,SAAVA,OAAgB,OAAO;QACxD,MAAMC,SAASD;QAEf,IAAI,iBAAiBC,UAAU,AAA8B,YAA9B,OAAOA,OAAO,WAAW,EAAe;YACrE,IAAI,CAAE,iBAAgBA,MAAK,KAAM,AAA6B,YAA7B,OAAOA,OAAO,UAAU,EACvD,OAAO;YAET,OAAO;QACT;QAEA,IAAI,YAAYA,UAAU,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EAAe;YAC3D,IAAI,CAAE,iBAAgBA,MAAK,KAAM,AAA6B,YAA7B,OAAOA,OAAO,UAAU,EACvD,OAAO;YAET,OAAO;QACT;QACA,OAAO;IACT;IAMA,IAAI,YAAoB;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,uCAAuC;IACpE;IAlIA,YAAoBC,EAAU,EAAEd,MAAc,EAAEE,UAAkB,CAAE;QARpE,uBAAQ,OAAR;QACA,uBAAQ,WAAR;QACA,uBAAQ,WAAR;QACA,uBAAQ,eAAR;QACA,uBAAQ,gBAAiD;QACzD,uBAAQ,kBAAgC;QACxC,uBAAQ,sBAAoC;QAG1C,IAAI,CAAC,GAAG,GAAGY;QACX,IAAI,CAAC,OAAO,GAAGd;QACf,IAAI,CAAC,OAAO,GAAGD,aAAaC;QAC5B,IAAI,CAAC,WAAW,GAAGE;IACrB;AA8HF"}

package/dist/es/service/index.mjs

@@ -0,0 +1,256 @@
+import { isAutoGLM } from "../ai-model/auto-glm/util.mjs";
+import { AIResponseParseError, AiExtractElementInfo, AiLocateElement, callAIWithObjectResponse } from "../ai-model/index.mjs";
+import { AiLocateSection } from "../ai-model/inspect.mjs";
+import { elementDescriberInstruction } from "../ai-model/prompt/describe.mjs";
+import { expandSearchArea } from "../common.mjs";
+import { ServiceError } from "../types.mjs";
+import { compositeElementInfoImg, cropByRect } from "@midscene/shared/img";
+import { getDebug } from "@midscene/shared/logger";
+import { assert } from "@midscene/shared/utils";
+import { createServiceDump } from "./utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('ai:service');
+class Service {
+    async locate(query, opt, modelConfig, abortSignal) {
+        const queryPrompt = 'string' == typeof query ? query : query.prompt;
+        assert(queryPrompt, 'query is required for locate');
+        assert('object' == typeof query, 'query should be an object for locate');
+        let searchAreaPrompt;
+        if (query.deepLocate) searchAreaPrompt = query.prompt;
+        const { modelFamily } = modelConfig;
+        if (searchAreaPrompt && !modelFamily) {
+            console.warn('The "deepLocate" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config');
+            searchAreaPrompt = void 0;
+        }
+        if (searchAreaPrompt && isAutoGLM(modelFamily)) {
+            console.warn('The "deepLocate" feature is not supported with AutoGLM.');
+            searchAreaPrompt = void 0;
+        }
+        const context = opt?.context || await this.contextRetrieverFn();
+        let searchArea;
+        let searchAreaRawResponse;
+        let searchAreaUsage;
+        let searchAreaResponse;
+        if (searchAreaPrompt) {
+            searchAreaResponse = await AiLocateSection({
+                context,
+                sectionDescription: searchAreaPrompt,
+                modelConfig,
+                abortSignal
+            });
+            assert(searchAreaResponse.rect, `cannot find search area for "${searchAreaPrompt}"${searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''}`);
+            searchAreaRawResponse = searchAreaResponse.rawResponse;
+            searchAreaUsage = searchAreaResponse.usage;
+            searchArea = searchAreaResponse.rect;
+        }
+        const startTime = Date.now();
+        const { parseResult, rect, rawResponse, usage, reasoning_content } = await AiLocateElement({
+            context,
+            targetElementDescription: queryPrompt,
+            searchConfig: searchAreaResponse,
+            modelConfig,
+            abortSignal
+        });
+        const timeCost = Date.now() - startTime;
+        const taskInfo = {
+            ...this.taskInfo ? this.taskInfo : {},
+            durationMs: timeCost,
+            rawResponse: JSON.stringify(rawResponse),
+            formatResponse: JSON.stringify(parseResult),
+            usage,
+            searchArea,
+            searchAreaRawResponse,
+            searchAreaUsage,
+            reasoning_content
+        };
+        let errorLog;
+        if (parseResult.errors?.length) errorLog = `failed to locate element: \n${parseResult.errors.join('\n')}`;
+        const dumpData = {
+            type: 'locate',
+            userQuery: {
+                element: queryPrompt
+            },
+            matchedElement: [],
+            matchedRect: rect,
+            data: null,
+            taskInfo,
+            deepLocate: !!searchArea,
+            error: errorLog
+        };
+        const elements = parseResult.elements || [];
+        const dump = createServiceDump({
+            ...dumpData,
+            matchedElement: elements
+        });
+        if (errorLog) throw new ServiceError(errorLog, dump);
+        if (elements.length > 1) throw new ServiceError(`locate: multiple elements found, length = ${elements.length}`, dump);
+        if (1 === elements.length) return {
+            element: {
+                center: elements[0].center,
+                rect: elements[0].rect,
+                description: elements[0].description
+            },
+            rect,
+            dump
+        };
+        return {
+            element: null,
+            rect,
+            dump
+        };
+    }
+    async extract(dataDemand, modelConfig, opt, pageDescription, multimodalPrompt, context) {
+        assert(context, 'context is required for extract');
+        assert('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
+        const startTime = Date.now();
+        let parseResult;
+        let rawResponse;
+        let usage;
+        let reasoning_content;
+        try {
+            const result = await AiExtractElementInfo({
+                context,
+                dataQuery: dataDemand,
+                multimodalPrompt,
+                extractOption: opt,
+                modelConfig,
+                pageDescription
+            });
+            parseResult = result.parseResult;
+            rawResponse = result.rawResponse;
+            usage = result.usage;
+            reasoning_content = result.reasoning_content;
+        } catch (error) {
+            if (error instanceof AIResponseParseError) {
+                const timeCost = Date.now() - startTime;
+                const taskInfo = {
+                    ...this.taskInfo ? this.taskInfo : {},
+                    durationMs: timeCost,
+                    rawResponse: error.rawResponse,
+                    usage: error.usage
+                };
+                const dump = createServiceDump({
+                    type: 'extract',
+                    userQuery: {
+                        dataDemand
+                    },
+                    matchedElement: [],
+                    data: null,
+                    taskInfo,
+                    error: error.message
+                });
+                throw new ServiceError(error.message, dump);
+            }
+            throw error;
+        }
+        const timeCost = Date.now() - startTime;
+        const taskInfo = {
+            ...this.taskInfo ? this.taskInfo : {},
+            durationMs: timeCost,
+            rawResponse,
+            formatResponse: JSON.stringify(parseResult),
+            usage,
+            reasoning_content
+        };
+        let errorLog;
+        if (parseResult.errors?.length) errorLog = `AI response error: \n${parseResult.errors.join('\n')}`;
+        const dumpData = {
+            type: 'extract',
+            userQuery: {
+                dataDemand
+            },
+            matchedElement: [],
+            data: null,
+            taskInfo,
+            error: errorLog
+        };
+        const { data, thought } = parseResult || {};
+        const dump = createServiceDump({
+            ...dumpData,
+            data
+        });
+        if (errorLog && !data) throw new ServiceError(errorLog, dump);
+        return {
+            data,
+            thought,
+            usage,
+            reasoning_content,
+            dump
+        };
+    }
+    async describe(target, modelConfig, opt) {
+        assert(target, 'target is required for service.describe');
+        const context = await this.contextRetrieverFn();
+        const { shotSize } = context;
+        const screenshotBase64 = context.screenshot.base64;
+        assert(screenshotBase64, 'screenshot is required for service.describe');
+        const { modelFamily } = modelConfig;
+        const systemPrompt = elementDescriberInstruction();
+        const defaultRectSize = 30;
+        const targetRect = Array.isArray(target) ? {
+            left: Math.floor(target[0] - defaultRectSize / 2),
+            top: Math.floor(target[1] - defaultRectSize / 2),
+            width: defaultRectSize,
+            height: defaultRectSize
+        } : target;
+        let imagePayload = await compositeElementInfoImg({
+            inputImgBase64: screenshotBase64,
+            size: shotSize,
+            elementsPositionInfo: [
+                {
+                    rect: targetRect
+                }
+            ],
+            borderThickness: 3
+        });
+        if (opt?.deepLocate) {
+            const searchArea = expandSearchArea(targetRect, shotSize);
+            debug('describe: cropping to searchArea', searchArea);
+            const croppedResult = await cropByRect(imagePayload, searchArea, 'qwen2.5-vl' === modelFamily);
+            imagePayload = croppedResult.imageBase64;
+        }
+        const msgs = [
+            {
+                role: 'system',
+                content: systemPrompt
+            },
+            {
+                role: 'user',
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: imagePayload,
+                            detail: 'high'
+                        }
+                    }
+                ]
+            }
+        ];
+        const res = await callAIWithObjectResponse(msgs, modelConfig);
+        const { content } = res;
+        assert(!content.error, `describe failed: ${content.error}`);
+        assert(content.description, 'failed to describe the element');
+        return content;
+    }
+    constructor(context, opt){
+        _define_property(this, "contextRetrieverFn", void 0);
+        _define_property(this, "taskInfo", void 0);
+        assert(context, 'context is required for Service');
+        if ('function' == typeof context) this.contextRetrieverFn = context;
+        else this.contextRetrieverFn = ()=>Promise.resolve(context);
+        if (void 0 !== opt?.taskInfo) this.taskInfo = opt.taskInfo;
+    }
+}
+export { Service as default };
+
+//# sourceMappingURL=index.mjs.map

package/dist/es/service/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"service/index.mjs","sources":["../../../src/service/index.ts"],"sourcesContent":["import { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport {\n  AIResponseParseError,\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport { type AIArgs, expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  DetailedLocateParam,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    let searchAreaPrompt;\n    if (query.deepLocate) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { modelFamily } = modelConfig;\n\n    if (searchAreaPrompt && !modelFamily) {\n      console.warn(\n        'The \"deepLocate\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    if (searchAreaPrompt && isAutoGLM(modelFamily)) {\n      console.warn('The \"deepLocate\" feature is not supported with AutoGLM.');\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n        abortSignal,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, rawResponse, usage, reasoning_content } =\n      await AiLocateElement({\n        context,\n        targetElementDescription: queryPrompt,\n        searchConfig: searchAreaResponse,\n        modelConfig,\n        abortSignal,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepLocate: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements = parseResult.elements || [];\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new ServiceError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          description: elements[0]!.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelConfig,\n        pageDescription,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          matchedElement: [],\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepLocate?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = await this.contextRetrieverFn();\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { modelFamily } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepLocate) {\n      const searchArea = expandSearchArea(targetRect, shotSize);\n      // Always crop in describe mode. Unlike locate's deepLocate (where\n      // cropping too small loses context for finding elements), describe's\n      // deepLocate intentionally zooms in so the model produces a more\n      // precise description from a focused view. expandSearchArea already\n      // guarantees a minimum 400x400 area with surrounding context.\n      debug('describe: cropping to searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        modelFamily === 'qwen2.5-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n      msgs,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["debug","getDebug","Service","query","opt","modelConfig","abortSignal","queryPrompt","assert","searchAreaPrompt","modelFamily","console","undefined","isAutoGLM","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","dump","createServiceDump","ServiceError","dataDemand","pageDescription","multimodalPrompt","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","content","Promise"],"mappings":";;;;;;;;;;;;;;;;;;;;AA2CA,MAAMA,QAAQC,SAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACzBC,WAAyB,EACM;QAC/B,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,OAAOD,aAAa;QAEpBC,OAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,IAAIM;QACJ,IAAIN,MAAM,UAAU,EAClBM,mBAAmBN,MAAM,MAAM;QAGjC,MAAM,EAAEO,WAAW,EAAE,GAAGL;QAExB,IAAII,oBAAoB,CAACC,aAAa;YACpCC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,IAAIH,oBAAoBI,UAAUH,cAAc;YAC9CC,QAAQ,IAAI,CAAC;YACbF,mBAAmBG;QACrB;QAEA,MAAME,UAAUV,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIW;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIT,kBAAkB;YACpBS,qBAAqB,MAAMC,gBAAgB;gBACzCL;gBACA,oBAAoBL;gBACpBJ;gBACAC;YACF;YACAE,OACEU,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAET,iBAAiB,CAAC,EAChDS,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAEC,iBAAiB,EAAE,GAChE,MAAMC,gBAAgB;YACpBb;YACA,0BAA0BP;YAC1B,cAAcW;YACdb;YACAC;QACF;QAEF,MAAMsB,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACR;YAC/BG;YACAV;YACAC;YACAC;YACAS;QACF;QAEA,IAAIK;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,4BAA4B,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAASzB;YACX;YACA,gBAAgB,EAAE;YAClB,aAAagB;YACb,MAAM;YACNM;YACA,YAAY,CAAC,CAACd;YACd,OAAOgB;QACT;QAEA,MAAME,WAAWX,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMY,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,aAAaL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,aACR,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,aAAaA,QAAQ,CAAC,EAAE,CAAE,WAAW;YACvC;YACAV;YACAW;QACF;QAGF,OAAO;YACL,SAAS;YACTX;YACAW;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/BhC,WAAyB,EACzBD,GAA0B,EAC1BkC,eAAwB,EACxBC,gBAAoC,EACpCzB,OAAmB,EACe;QAClCN,OAAOM,SAAS;QAChBN,OACE,AAAsB,YAAtB,OAAO6B,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAMjB,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMc,SAAS,MAAMC,qBAAwB;gBAC3C3B;gBACA,WAAWuB;gBACXE;gBACA,eAAenC;gBACfC;gBACAiC;YACF;YACAhB,cAAckB,OAAO,WAAW;YAChChB,cAAcgB,OAAO,WAAW;YAChCf,QAAQe,OAAO,KAAK;YACpBd,oBAAoBc,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,sBAAsB;gBAEzC,MAAMf,WAAWP,KAAK,GAAG,KAAKD;gBAC9B,MAAMS,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAac,MAAM,WAAW;oBAC9B,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMR,OAAOC,kBAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEE;oBAAW;oBACxB,gBAAgB,EAAE;oBAClB,MAAM;oBACNR;oBACA,OAAOa,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIN,aAAaM,MAAM,OAAO,EAAER;YACxC;YACA,MAAMQ;QACR;QAEA,MAAMd,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZJ;YACA,gBAAgBM,KAAK,SAAS,CAACR;YAC/BG;YACAC;QACF;QAEA,IAAIK;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,qBAAqB,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNR;YACA,OAAOE;QACT;QAEA,MAAM,EAAEa,IAAI,EAAEC,OAAO,EAAE,GAAGvB,eAAe,CAAC;QAE1C,MAAMY,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACXY;QACF;QAEA,IAAIb,YAAY,CAACa,MACf,MAAM,IAAIR,aAAaL,UAAUG;QAGnC,OAAO;YACLU;YACAC;YACApB;YACAC;YACAQ;QACF;IACF;IAEA,MAAM,SACJY,MAA+B,EAC/BzC,WAAyB,EACzBD,GAEC,EACwD;QACzDI,OAAOsC,QAAQ;QACf,MAAMhC,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEiC,QAAQ,EAAE,GAAGjC;QACrB,MAAMkC,mBAAmBlC,QAAQ,UAAU,CAAC,MAAM;QAClDN,OAAOwC,kBAAkB;QAEzB,MAAM,EAAEtC,WAAW,EAAE,GAAGL;QACxB,MAAM4C,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAIhD,KAAK,YAAY;YACnB,MAAMW,aAAa0C,iBAAiBL,YAAYL;YAMhD/C,MAAM,oCAAoCe;YAC1C,MAAM2C,gBAAgB,MAAMC,WAC1BJ,cACAxC,YACAL,AAAgB,iBAAhBA;YAEF6C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,MAAM,MAAMC,yBAChBF,MACAvD;QAGF,MAAM,EAAE0D,OAAO,EAAE,GAAGF;QACpBrD,OAAO,CAACuD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DvD,OAAOuD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IA9UA,YACEjD,OAA2D,EAC3DV,GAAoB,CACpB;QAPF;QAEA;QAMEI,OAAOM,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMkD,QAAQ,OAAO,CAAClD;QAGlD,IAAI,AAAyB,WAAlBV,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAiUF"}

package/dist/es/service/utils.mjs

@@ -0,0 +1,15 @@
+import { uuid } from "@midscene/shared/utils";
+function createServiceDump(data) {
+    const baseData = {
+        logTime: Date.now()
+    };
+    const finalData = {
+        logId: uuid(),
+        ...baseData,
+        ...data
+    };
+    return finalData;
+}
+export { createServiceDump };
+
+//# sourceMappingURL=utils.mjs.map

package/dist/es/service/utils.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"service/utils.mjs","sources":["../../../src/service/utils.ts"],"sourcesContent":["import type { DumpMeta, PartialServiceDumpFromSDK, ServiceDump } from '@/types';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function createServiceDump(\n  data: PartialServiceDumpFromSDK,\n): ServiceDump {\n  const baseData: DumpMeta = {\n    logTime: Date.now(),\n  };\n  const finalData: ServiceDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  return finalData;\n}\n"],"names":["createServiceDump","data","baseData","Date","finalData","uuid"],"mappings":";AAGO,SAASA,kBACdC,IAA+B;IAE/B,MAAMC,WAAqB;QACzB,SAASC,KAAK,GAAG;IACnB;IACA,MAAMC,YAAyB;QAC7B,OAAOC;QACP,GAAGH,QAAQ;QACX,GAAGD,IAAI;IACT;IAEA,OAAOG;AACT"}

package/dist/es/skill/index.mjs

@@ -0,0 +1,38 @@
+import { CLIError, runToolsCLI } from "@midscene/shared/cli";
+import { BaseMidsceneTools } from "@midscene/shared/mcp";
+import { Agent } from "../agent/agent.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class SkillMidsceneTools extends BaseMidsceneTools {
+    createTemporaryDevice() {
+        return new this.DeviceClass();
+    }
+    async ensureAgent() {
+        if (!this.agent) {
+            const device = new this.DeviceClass();
+            this.agent = new Agent(device);
+        }
+        return this.agent;
+    }
+    constructor(DeviceClass){
+        super(), _define_property(this, "DeviceClass", void 0), this.DeviceClass = DeviceClass;
+    }
+}
+function runSkillCLI(options) {
+    const tools = new SkillMidsceneTools(options.DeviceClass);
+    return runToolsCLI(tools, options.scriptName).catch((e)=>{
+        if (!(e instanceof CLIError)) console.error(e);
+        process.exit(e instanceof CLIError ? e.exitCode : 1);
+    });
+}
+export { runSkillCLI };
+
+//# sourceMappingURL=index.mjs.map

package/dist/es/skill/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"skill/index.mjs","sources":["../../../src/skill/index.ts"],"sourcesContent":["import { CLIError, runToolsCLI } from '@midscene/shared/cli';\nimport { BaseMidsceneTools } from '@midscene/shared/mcp';\nimport type { BaseAgent, BaseDevice } from '@midscene/shared/mcp';\nimport { Agent } from '../agent/agent';\nimport type { AbstractInterface } from '../device';\n\ntype DeviceClass = new (...args: any[]) => AbstractInterface;\n\n/**\n * Skill tools manager that lazily creates Agent from a Device class.\n * Used by runSkillCLI for CLI / Agent Skills scenarios where no agent exists at startup.\n */\nclass SkillMidsceneTools extends BaseMidsceneTools<BaseAgent> {\n  constructor(private DeviceClass: DeviceClass) {\n    super();\n  }\n\n  protected createTemporaryDevice(): BaseDevice {\n    return new this.DeviceClass() as unknown as BaseDevice;\n  }\n\n  protected async ensureAgent(): Promise<BaseAgent> {\n    if (!this.agent) {\n      const device = new this.DeviceClass();\n      this.agent = new Agent(device) as unknown as BaseAgent;\n    }\n    return this.agent;\n  }\n}\n\nexport interface SkillCLIOptions {\n  scriptName: string;\n  DeviceClass: DeviceClass;\n}\n\n/**\n * Launch a Skill CLI for a custom interface Device class.\n * This enables AI coding assistants (Claude Code, Cline, etc.) to control\n * your custom interface through CLI commands.\n *\n * @example\n * ```typescript\n * #!/usr/bin/env node\n * import { runSkillCLI } from '@midscene/core/skill';\n * import { SampleDevice } from './sample-device';\n *\n * runSkillCLI({\n *   DeviceClass: SampleDevice,\n *   scriptName: 'my-device',\n * });\n * ```\n */\nexport function runSkillCLI(options: SkillCLIOptions): Promise<void> {\n  const tools = new SkillMidsceneTools(options.DeviceClass);\n  return runToolsCLI(tools, options.scriptName).catch((e) => {\n    if (!(e instanceof CLIError)) console.error(e);\n    process.exit(e instanceof CLIError ? e.exitCode : 1);\n  });\n}\n"],"names":["SkillMidsceneTools","BaseMidsceneTools","device","Agent","DeviceClass","runSkillCLI","options","tools","runToolsCLI","e","CLIError","console","process"],"mappings":";;;;;;;;;;;;;AAYA,MAAMA,2BAA2BC;IAKrB,wBAAoC;QAC5C,OAAO,IAAI,IAAI,CAAC,WAAW;IAC7B;IAEA,MAAgB,cAAkC;QAChD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAMC,SAAS,IAAI,IAAI,CAAC,WAAW;YACnC,IAAI,CAAC,KAAK,GAAG,IAAIC,MAAMD;QACzB;QACA,OAAO,IAAI,CAAC,KAAK;IACnB;IAdA,YAAoBE,WAAwB,CAAE;QAC5C,KAAK,wDADaA,WAAW,GAAXA;IAEpB;AAaF;AAwBO,SAASC,YAAYC,OAAwB;IAClD,MAAMC,QAAQ,IAAIP,mBAAmBM,QAAQ,WAAW;IACxD,OAAOE,YAAYD,OAAOD,QAAQ,UAAU,EAAE,KAAK,CAAC,CAACG;QACnD,IAAI,CAAEA,CAAAA,aAAaC,QAAO,GAAIC,QAAQ,KAAK,CAACF;QAC5CG,QAAQ,IAAI,CAACH,aAAaC,WAAWD,EAAE,QAAQ,GAAG;IACpD;AACF"}