@midscene/core 1.2.2-beta-20260115034338.0 → 1.2.2-beta-20260115090041.0

package/dist/types/ai-model/auto-glm/actions.d.ts

@@ -0,0 +1,77 @@
+import type { PlanningAction } from '../../types';
+export interface BaseAction {
+    _metadata: string;
+    think?: string;
+}
+export interface TapAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Tap';
+    element: [number, number];
+}
+export interface DoubleTapAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Double Tap';
+    element: [number, number];
+}
+export interface TypeAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Type';
+    text: string;
+}
+export interface SwipeAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Swipe';
+    start: [number, number];
+    end: [number, number];
+}
+export interface LongPressAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Long Press';
+    element: [number, number];
+}
+export interface LaunchAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Launch';
+    app: string;
+}
+export interface BackAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Back';
+}
+export interface HomeAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Home';
+}
+export interface WaitAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Wait';
+    durationMs: number;
+}
+export interface InteractAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Interact';
+}
+export interface CallAPIAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Call_API';
+    instruction: string;
+}
+export interface TakeoverAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Take_over';
+    message: string;
+}
+export interface NoteAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Note';
+    message: string;
+}
+export interface FinishAction extends BaseAction {
+    _metadata: 'finish';
+    message: string;
+}
+export type ParsedAction = TapAction | DoubleTapAction | TypeAction | SwipeAction | LongPressAction | LaunchAction | BackAction | HomeAction | WaitAction | InteractAction | CallAPIAction | TakeoverAction | NoteAction | FinishAction;
+export declare function transformAutoGLMAction(action: ParsedAction, size: {
+    width: number;
+    height: number;
+}): PlanningAction[];

package/dist/types/ai-model/auto-glm/index.d.ts

@@ -0,0 +1,6 @@
+export { getAutoGLMLocatePrompt, getAutoGLMPlanPrompt } from './prompt';
+export { parseAutoGLMLocateResponse, parseAutoGLMResponse, parseAction, } from './parser';
+export { autoGLMPlanning } from './planning';
+export { transformAutoGLMAction } from './actions';
+export { autoGLMCoordinateToBbox, isAutoGLM } from './util';
+export type { ParsedAction } from './actions';

package/dist/types/ai-model/auto-glm/parser.d.ts

@@ -0,0 +1,18 @@
+import type { ParsedAction } from './actions';
+export declare const extractValueAfter: (src: string, key: string) => string;
+export declare function parseAction(response: {
+    think: string;
+    content: string;
+}): ParsedAction;
+export declare function parseAutoGLMResponse(content: string): {
+    think: string;
+    content: string;
+};
+export declare function parseAutoGLMLocateResponse(rawResponse: string): {
+    think: string;
+    coordinates: {
+        x: number;
+        y: number;
+    } | null;
+    error?: string;
+};

package/dist/types/ai-model/auto-glm/planning.d.ts

@@ -0,0 +1,9 @@
+import type { PlanningAIResponse, UIContext } from '../../types';
+import type { IModelConfig } from '@midscene/shared/env';
+import type { ConversationHistory } from '../conversation-history';
+export declare function autoGLMPlanning(userInstruction: string, options: {
+    conversationHistory: ConversationHistory;
+    context: UIContext;
+    modelConfig: IModelConfig;
+    actionContext?: string;
+}): Promise<PlanningAIResponse>;

package/dist/types/ai-model/auto-glm/prompt.d.ts

@@ -0,0 +1,27 @@
+/**
+ * Auto-GLM Prompt Templates
+ *
+ * Portions of this file are derived from Open-AutoGLM
+ * Copyright (c) 2024 zai-org
+ * Licensed under the Apache License, Version 2.0
+ *
+ * Source: https://github.com/zai-org/Open-AutoGLM
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Modifications:
+ * - Adapted prompts for Midscene.js integration
+ */
+import type { TVlModeTypes } from '@midscene/shared/env';
+export declare const getAutoGLMPlanPrompt: (vlMode: TVlModeTypes | undefined) => string;
+export declare const getAutoGLMLocatePrompt: (vlMode: TVlModeTypes | undefined) => string;

package/dist/types/ai-model/auto-glm/util.d.ts

@@ -0,0 +1,16 @@
+import type { TVlModeTypes } from '@midscene/shared/env';
+/**
+ * Auto-GLM coordinate system range: [0, AUTO_GLM_COORDINATE_MAX]
+ */
+export declare const AUTO_GLM_COORDINATE_MAX = 1000;
+/**
+ * Check if the vlMode is auto-glm or auto-glm-multilingual
+ * @param vlMode The VL mode to check
+ * @returns true if vlMode is auto-glm or auto-glm-multilingual
+ */
+export declare function isAutoGLM(vlMode: TVlModeTypes | undefined): boolean;
+/**
+ * Convert auto-glm coordinate [0,999] to bbox
+ * Auto-glm uses [0,999] coordinate system, maps to image size, and creates a 10x10 bbox around the point
+ */
+export declare function autoGLMCoordinateToBbox(x: number, y: number, width: number, height: number): [number, number, number, number];

package/dist/types/ai-model/index.d.ts

@@ -6,6 +6,7 @@ export { generateYamlTest, generateYamlTestStream, } from './prompt/yaml-generat
 export type { ChatCompletionMessageParam } from 'openai/resources/index';
 export { AiLocateElement, AiExtractElementInfo, AiLocateSection, AiJudgeOrderSensitive, } from './inspect';
 export { plan } from './llm-planning';
+export { autoGLMPlanning } from './auto-glm/planning';
 export { adaptBboxToRect } from '../common';
 export { uiTarsPlanning, resizeImageForUiTars } from './ui-tars-planning';
 export { ConversationHistory, type ConversationHistoryOptions, } from './conversation-history';

package/dist/types/ai-model/latest-locate-recorder.d.ts

@@ -0,0 +1,14 @@
+interface ILocate {
+    prompt: string;
+    bbox: [number, number, number, number];
+}
+export declare class LatestLocateRecorder {
+    latestLocate: ILocate | undefined;
+    source: string;
+    recordLocate(locate: ILocate, source: string): void;
+    getLatestLocate(): {
+        locate: ILocate | undefined;
+        source: string;
+    };
+}
+export {};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.2.2-beta-20260115034338.0",
+  "version": "1.2.2-beta-20260115090041.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -89,7 +89,7 @@
     "semver": "7.5.2",
     "undici": "^6.0.0",
     "zod": "3.24.3",
-    "@midscene/shared": "1.2.2-beta-20260115034338.0"
+    "@midscene/shared": "1.2.2-beta-20260115090041.0"
   },
   "devDependencies": {
     "@rslib/core": "^0.18.3",