@aiscene/core 9.0.6 → 9.0.8

package/dist/types/ai-model/model-adapter/types.d.ts

@@ -0,0 +1,218 @@
+import type { PixelBbox, PlanningAction } from '../../types';
+import type { AIUsageInfo } from '../../types';
+import type { IModelConfig, TIntent, TModelReasoningEnabled, TModelResponseFormat } from '@aiscene/shared/env';
+import type OpenAI from 'openai';
+import type { JsonParser, JsonParserContext, JsonParserSource } from '../shared/json';
+import type { LocateResultCodec, LocateResultFormatDefinition, ResolvedLocateResultCoordinates } from '../shared/model-locate-result/types';
+import type { LocateFn } from '../workflows/grounding/types';
+import type { PlanFn } from '../workflows/planning/types';
+import type { CustomPlanningDefinition } from './custom-planning-types';
+import type { ImagePreprocessPolicy } from './image-preprocess';
+import type { InsightAdapter, InsightDefinition } from './insight-protocol';
+import type { StandardLocateProtocol, StandardLocateProtocolDefinition } from './locate-protocol';
+import type { StandardPlanningProtocol, StandardPlanningProtocolDefinition } from './planning-protocol';
+export type { ImagePreprocessPolicy, JsonParser, JsonParserContext, JsonParserSource, };
+export type JsonParserPreset = 'lenient-json';
+export interface ReasoningInput {
+    reasoningEnabled?: TModelReasoningEnabled;
+    reasoningEffort?: string;
+    reasoningBudget?: number;
+}
+export interface ChatCompletionParamsResult {
+    config: Record<string, unknown>;
+}
+export interface MidsceneChatCompletionDefaults {
+    temperature: number;
+}
+export interface ChatCompletionCallUserConfig extends ReasoningInput {
+    temperature?: number;
+    responseFormat?: TModelResponseFormat;
+}
+export type ChatCompletionUnsupportedUserConfig = keyof ChatCompletionCallUserConfig;
+export interface ChatCompletionCallInput {
+    intent?: TIntent;
+    userConfig?: ChatCompletionCallUserConfig;
+    /**
+     * Number of preceding semantic parsing failures for this request.
+     * This is execution context, not part of the user's model configuration.
+     */
+    semanticRetryAttempt?: number;
+    requiresOriginalImageDetail?: boolean;
+    /**
+     * Whether this call expects a JSON object response.
+     *
+     * This must not be inferred from `intent === 'default'`: intent selects a
+     * model-config slot, while many non-locate calls (for example, browser
+     * extension recording data generation that produces YAML) also use the
+     * default model.
+     */
+    expectedJsonObjectResponse?: boolean;
+}
+export interface ChatCompletionCallContext {
+    intent?: TIntent;
+    userConfig: ChatCompletionCallUserConfig;
+    semanticRetryAttempt?: number;
+    requiresOriginalImageDetail?: boolean;
+    expectedJsonObjectResponse?: boolean;
+    midsceneDefaults: MidsceneChatCompletionDefaults;
+}
+export type ImageDetail = 'auto' | 'low' | 'high' | 'original';
+export interface ContentAndReasoning {
+    content: string;
+    reasoning_content: string;
+}
+export type ChatCompletionContentSource = (OpenAI.Chat.Completions.ChatCompletionMessage & {
+    reasoning_content?: string;
+}) | (OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta & {
+    reasoning_content?: string;
+});
+export type ExtractContentAndReasoning = (message: ChatCompletionContentSource | undefined) => ContentAndReasoning;
+export interface ChatCompletionAdapter {
+    unsupportedUserConfig: ChatCompletionUnsupportedUserConfig[];
+    buildChatCompletionParams(input: ChatCompletionCallInput): ChatCompletionParamsResult;
+    resolveImageDetail(input: ChatCompletionCallInput): ImageDetail | undefined;
+    extractContentAndReasoning: ExtractContentAndReasoning;
+    useReasoningAsContentFallback: boolean;
+    replayRawAssistantMessage: boolean;
+}
+type ChatCompletionMessageExtraction = {
+    messageExtraction?: {
+        kind: 'default';
+        reasoningContentKeys?: string[];
+    };
+} | {
+    messageExtraction: {
+        kind: 'custom';
+        extractContentAndReasoning: ExtractContentAndReasoning;
+    };
+};
+export type ChatCompletionDefinition = ChatCompletionMessageExtraction & {
+    unsupportedUserConfig?: ChatCompletionUnsupportedUserConfig[];
+    buildChatCompletionParams?: (input: ChatCompletionCallContext) => ChatCompletionParamsResult;
+    resolveImageDetail?: (input: ChatCompletionCallContext) => ImageDetail | undefined;
+    useReasoningAsContentFallback?: boolean;
+    /**
+     * Replay the provider's original assistant message in later planning turns.
+     *
+     * Enable this only for model families whose API requires opaque response
+     * fields (for example, reasoning state or thought signatures) to be passed
+     * back unchanged. The default replays Midscene's normalized assistant text.
+     */
+    replayRawAssistantMessage?: boolean;
+};
+export type ImagePreprocessDefinition = Partial<ImagePreprocessPolicy>;
+interface PlanningPolicy {
+    cacheEnabled: boolean;
+    defaultReplanningCycleLimit: number;
+    /**
+     * Whether aiAct can use planning action coordinates as the first-stage
+     * search area for deepLocate.
+     *
+     * Custom planning models may return only action coordinates without a target
+     * element description. Those results can be used as direct plan hits, but
+     * cannot drive deepLocate's second-stage locate call because that call also
+     * needs a query prompt describing the target element.
+     */
+    supportsActionDeepLocate: boolean;
+}
+export type PlanningAdapter = (PlanningPolicy & {
+    kind: 'standard';
+    protocol: StandardPlanningProtocol;
+    locateResultCodec?: LocateResultCodec;
+}) | (PlanningPolicy & {
+    kind: 'custom';
+    planFn: PlanFn;
+    coordinateSystem?: ResolvedLocateResultCoordinates;
+});
+export type PlanningDefinition = (Partial<PlanningPolicy> & {
+    kind?: 'standard';
+    protocol?: StandardPlanningProtocolDefinition;
+    locateResultFormat?: LocateResultFormatDefinition | false;
+}) | (Partial<PlanningPolicy> & ({
+    kind: 'custom';
+    planner: CustomPlanningDefinition<any>;
+    planFn?: never;
+} | {
+    kind: 'custom';
+    planFn: PlanFn;
+    planner?: never;
+}));
+export type LocateUserMessageContentOrder = 'image-first' | 'prompt-first';
+type StandardLocateAdapter = {
+    kind: 'standard';
+    userMessageContentOrder: LocateUserMessageContentOrder;
+    element: LocateOperation;
+    searchArea?: LocateOperation;
+};
+interface LocateOperation {
+    protocol: StandardLocateProtocol;
+    resultCodec: LocateResultCodec;
+}
+type CustomLocateAdapter = {
+    kind: 'custom';
+    locateFn: LocateFn;
+};
+export type LocateAdapter = StandardLocateAdapter | CustomLocateAdapter;
+type StandardLocateDefinition = {
+    kind?: 'standard';
+    userMessageContentOrder?: LocateUserMessageContentOrder;
+    element?: LocateOperationDefinition;
+    searchArea?: LocateOperationDefinition | false;
+};
+interface LocateOperationDefinition {
+    protocol?: StandardLocateProtocolDefinition;
+    resultFormat?: LocateResultFormatDefinition;
+}
+export interface PlanningTapLocatorDefinition {
+    buildSystemPrompt(): string;
+    getLocatedPixelBbox(actions: PlanningAction[]): PixelBbox | undefined;
+}
+type CustomLocateDefinition = {
+    kind: 'custom';
+} & ({
+    locateFn: LocateFn;
+    planningTapLocator?: never;
+} | {
+    planningTapLocator: PlanningTapLocatorDefinition;
+    locateFn?: never;
+});
+export type LocateDefinition = StandardLocateDefinition | CustomLocateDefinition;
+export interface ModelAdapter {
+    jsonParser: JsonParser;
+    chatCompletion: ChatCompletionAdapter;
+    acceptBbox2dAlias: boolean;
+    imagePreprocess: ImagePreprocessPolicy;
+    insight: InsightAdapter;
+    planning: PlanningAdapter;
+    locate: LocateAdapter;
+}
+export interface ModelRuntime {
+    config: IModelConfig;
+    adapter: ModelAdapter;
+    /**
+     * Report execution that owns this model runtime. It is carried on a
+     * per-execution runtime copy so concurrent Agent operations never share it.
+     */
+    executionId?: string;
+    /**
+     * Optional callback fired after every underlying model call with the shaped
+     * usage info. Provides a single collection point for callers that want to
+     * aggregate usage across all model invocations (including auxiliary calls
+     * such as order-sensitive judging and deep-locate search-area calls).
+     */
+    onUsage?: (usage: AIUsageInfo) => void;
+}
+export interface ModelAdapterDefinition {
+    jsonParser?: JsonParserPreset | JsonParser;
+    chatCompletion?: ChatCompletionDefinition;
+    /**
+     * Temporary compatibility for models that may occasionally return
+     * `bbox_2d` instead of `bbox`. Currently enabled only by Qwen adapters and
+     * should be removed once this model behavior no longer needs accommodation.
+     */
+    acceptBbox2dAlias?: boolean;
+    imagePreprocess?: ImagePreprocessDefinition;
+    insight?: InsightDefinition;
+    planning?: PlanningDefinition;
+    locate?: LocateDefinition;
+}

package/dist/types/ai-model/models/auto-glm/actions.d.ts

@@ -0,0 +1,81 @@
+import type { DeviceAction } from '../../../device';
+import type { PlanningAction } from '../../../types';
+import type { CoordinateDistanceAxis } from '../../shared/model-locate-result';
+type CoordinateDistanceToPixels = (delta: number, axis: CoordinateDistanceAxis) => number;
+export interface BaseAction {
+    _metadata: string;
+    think?: string;
+}
+export interface TapAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Tap';
+    element: [number, number];
+}
+export interface DoubleTapAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Double Tap';
+    element: [number, number];
+}
+export interface TypeAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Type';
+    text: string;
+}
+export interface SwipeAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Swipe';
+    start: [number, number];
+    end: [number, number];
+}
+export interface LongPressAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Long Press';
+    element: [number, number];
+}
+export interface LaunchAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Launch';
+    app: string;
+}
+export interface BackAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Back';
+}
+export interface HomeAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Home';
+}
+export interface WaitAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Wait';
+    durationMs: number;
+}
+export interface InteractAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Interact';
+}
+export interface CallAPIAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Call_API';
+    instruction: string;
+}
+export interface TakeoverAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Take_over';
+    message: string;
+}
+export interface NoteAction extends BaseAction {
+    _metadata: 'do';
+    action: 'Note';
+    message: string;
+}
+export interface FinishAction extends BaseAction {
+    _metadata: 'finish';
+    message: string;
+}
+export type AutoGLMParsedAction = TapAction | DoubleTapAction | TypeAction | SwipeAction | LongPressAction | LaunchAction | BackAction | HomeAction | WaitAction | InteractAction | CallAPIAction | TakeoverAction | NoteAction | FinishAction;
+export declare function transformAutoGLMAction(action: AutoGLMParsedAction, { actionSpace, coordinateDistanceToPixels, }: {
+    actionSpace?: DeviceAction[];
+    coordinateDistanceToPixels: CoordinateDistanceToPixels;
+}): PlanningAction[];
+export {};

package/dist/types/ai-model/models/auto-glm/adapter.d.ts

@@ -0,0 +1,5 @@
+import type { ModelAdapterDefinition } from '../../model-adapter/types';
+export declare const autoGlmAdapters: {
+    'auto-glm': ModelAdapterDefinition;
+    'auto-glm-multilingual': ModelAdapterDefinition;
+};

package/dist/types/ai-model/models/auto-glm/locate.d.ts

@@ -0,0 +1,2 @@
+import type { PlanningTapLocatorDefinition } from '../../model-adapter/types';
+export declare function createAutoGlmPlanningTapLocator(isMultilingual: boolean): PlanningTapLocatorDefinition;

package/dist/types/ai-model/models/auto-glm/parser.d.ts

@@ -0,0 +1,14 @@
+import type { AutoGLMParsedAction } from './actions';
+export declare const extractValueAfter: (src: string, key: string) => string;
+export declare function parseAutoGLMPlanningAction(response: {
+    think: string;
+    content: string;
+}): AutoGLMParsedAction;
+export declare function parseAutoGLMResponse(content: string): {
+    think: string;
+    content: string;
+};
+export declare function parseAutoGLMPlanningResponse(content: string): {
+    response: ReturnType<typeof parseAutoGLMResponse>;
+    action: AutoGLMParsedAction;
+};

package/dist/types/ai-model/models/auto-glm/planning.d.ts

@@ -0,0 +1,5 @@
+import type { CustomPlanningDefinition } from '../../model-adapter/custom-planning-types';
+import { parseAutoGLMPlanningResponse } from './parser';
+type AutoGLMParsedResponse = ReturnType<typeof parseAutoGLMPlanningResponse>;
+export declare function createAutoGlmPlanner(isMultilingual: boolean): CustomPlanningDefinition<AutoGLMParsedResponse>;
+export {};

package/dist/types/ai-model/models/auto-glm/prompt.d.ts

@@ -0,0 +1,4 @@
+export declare const getAutoGLMMultilingualPlanPrompt: () => string;
+export declare const getAutoGLMChinesePlanPrompt: () => string;
+export declare const getAutoGLMMultilingualLocatePrompt: () => string;
+export declare const getAutoGLMChineseLocatePrompt: () => string;

package/dist/types/ai-model/models/deepseek/adapter.d.ts

@@ -0,0 +1,39 @@
+import type { ChatCompletionCallContext, ChatCompletionParamsResult } from '../../model-adapter/types';
+import { type LocateResultValue } from '../../shared/model-locate-result';
+declare function parseDeepSeekPointLocateValue(input: unknown): LocateResultValue;
+declare function parseDeepSeekBboxLocateValue(input: unknown): LocateResultValue;
+export declare const deepSeekAdapters: {
+    deepseek: {
+        chatCompletion: {
+            unsupportedUserConfig: "reasoningBudget"[];
+            buildChatCompletionParams: (input: ChatCompletionCallContext) => ChatCompletionParamsResult;
+            useReasoningAsContentFallback: true;
+            replayRawAssistantMessage: false;
+        };
+        locate: {
+            element: {
+                protocol: import("../../model-adapter/locate-protocol").StandardLocateProtocol;
+                resultFormat: {
+                    coordinates: {
+                        readonly shape: "point";
+                        readonly order: "xy";
+                        readonly normalizedBy: 1000;
+                    };
+                    parseRawLocateValue: typeof parseDeepSeekPointLocateValue;
+                };
+            };
+            searchArea: {
+                protocol: import("../../model-adapter/locate-protocol").StandardLocateProtocol;
+                resultFormat: {
+                    coordinates: {
+                        readonly shape: "bbox";
+                        readonly order: "xy";
+                        readonly normalizedBy: 1000;
+                    };
+                    parseRawLocateValue: typeof parseDeepSeekBboxLocateValue;
+                };
+            };
+        };
+    };
+};
+export {};

package/dist/types/ai-model/models/deepseek/locate-protocol.d.ts

@@ -0,0 +1,5 @@
+import type { ParsedLocateResponse, StandardLocateProtocol } from '../../model-adapter/locate-protocol';
+export declare function parseDeepSeekLocateOutput(content: string): ParsedLocateResponse;
+export declare function parseDeepSeekSearchAreaOutput(content: string): ParsedLocateResponse;
+export declare const deepSeekElementLocateProtocol: StandardLocateProtocol;
+export declare const deepSeekSearchAreaProtocol: StandardLocateProtocol;

package/dist/types/ai-model/models/default.d.ts

@@ -0,0 +1,2 @@
+import type { ModelAdapterDefinition } from '../model-adapter/types';
+export declare const defaultOpenAICompatibleAdapterConfig: ModelAdapterDefinition;

package/dist/types/ai-model/models/doubao.d.ts

@@ -0,0 +1,7 @@
+import type { ModelAdapterDefinition } from '../model-adapter/types';
+import { type LocateResultValue } from '../shared/model-locate-result';
+export declare function parseDoubaoRawLocateValue(input: unknown): LocateResultValue;
+export declare const doubaoAdapters: {
+    'doubao-vision': ModelAdapterDefinition;
+    'doubao-seed': ModelAdapterDefinition;
+};

package/dist/types/ai-model/models/gemini.d.ts

@@ -0,0 +1,37 @@
+import type OpenAI from 'openai';
+import type { ChatCompletionCallContext, ChatCompletionContentSource, ChatCompletionParamsResult, ContentAndReasoning } from '../model-adapter/types';
+type GeminiContentPart = Record<string, unknown> & {
+    text?: string;
+    thought?: boolean;
+};
+type GeminiContentExtension = {
+    content: string | null | GeminiContentPart[];
+    reasoning_content?: string;
+    extra_content?: unknown;
+};
+type GeminiContentSource = ChatCompletionContentSource | (Omit<OpenAI.Chat.Completions.ChatCompletionMessage, 'content'> & GeminiContentExtension) | (Omit<OpenAI.Chat.Completions.ChatCompletionChunk.Choice.Delta, 'content'> & GeminiContentExtension);
+export declare const extractGeminiContentAndReasoning: (message: GeminiContentSource | undefined) => ContentAndReasoning;
+export declare const geminiAdapters: {
+    gemini: {
+        chatCompletion: {
+            unsupportedUserConfig: "reasoningBudget"[];
+            buildChatCompletionParams: (input: ChatCompletionCallContext) => ChatCompletionParamsResult;
+            messageExtraction: {
+                kind: "custom";
+                extractContentAndReasoning: (message: GeminiContentSource | undefined) => ContentAndReasoning;
+            };
+        };
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: {
+                        shape: "bbox";
+                        order: "yx";
+                        normalizedBy: number;
+                    };
+                };
+            };
+        };
+    };
+};
+export {};

package/dist/types/ai-model/models/glm.d.ts

@@ -0,0 +1,21 @@
+import type { ChatCompletionCallContext, ChatCompletionParamsResult } from '../model-adapter/types';
+export declare const glmAdapters: {
+    'glm-v': {
+        chatCompletion: {
+            unsupportedUserConfig: ("reasoningEffort" | "reasoningBudget")[];
+            buildChatCompletionParams: (input: ChatCompletionCallContext) => ChatCompletionParamsResult;
+            useReasoningAsContentFallback: true;
+        };
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: {
+                        shape: "bbox";
+                        order: "xy";
+                        normalizedBy: number;
+                    };
+                };
+            };
+        };
+    };
+};

package/dist/types/ai-model/models/gpt.d.ts

@@ -0,0 +1,20 @@
+import type { ChatCompletionCallContext, ChatCompletionParamsResult, ImageDetail } from '../model-adapter/types';
+export declare const gptAdapters: {
+    'gpt-5': {
+        chatCompletion: {
+            unsupportedUserConfig: "reasoningBudget"[];
+            buildChatCompletionParams: (input: ChatCompletionCallContext) => ChatCompletionParamsResult;
+            resolveImageDetail: (input: ChatCompletionCallContext) => ImageDetail | undefined;
+        };
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: {
+                        shape: "bbox";
+                        order: "xy";
+                    };
+                };
+            };
+        };
+    };
+};

package/dist/types/ai-model/models/index.d.ts

@@ -0,0 +1,2 @@
+export { getModelAdapter, getModelRuntime, } from './registry';
+export type { ModelRuntime } from '../model-adapter/types';

package/dist/types/ai-model/models/kimi.d.ts

@@ -0,0 +1,45 @@
+import type { ChatCompletionCallContext, ChatCompletionParamsResult } from '../model-adapter/types';
+import { type LocateResultValue } from '../shared/model-locate-result';
+declare function parseKimiRawLocateValue(input: unknown): LocateResultValue;
+export declare const kimiAdapters: {
+    kimi: {
+        chatCompletion: {
+            unsupportedUserConfig: ("reasoningEffort" | "reasoningBudget")[];
+            buildChatCompletionParams: (input: ChatCompletionCallContext) => ChatCompletionParamsResult;
+            useReasoningAsContentFallback: true;
+        };
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: {
+                        readonly shape: "point";
+                        readonly order: "xy";
+                        readonly normalizedBy: 1;
+                    };
+                    parseRawLocateValue: typeof parseKimiRawLocateValue;
+                };
+            };
+        };
+    };
+    kimi3: {
+        chatCompletion: {
+            unsupportedUserConfig: ("reasoningEnabled" | "reasoningBudget")[];
+            buildChatCompletionParams: (input: ChatCompletionCallContext) => ChatCompletionParamsResult;
+            useReasoningAsContentFallback: true;
+            replayRawAssistantMessage: true;
+        };
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: {
+                        readonly shape: "point";
+                        readonly order: "xy";
+                        readonly normalizedBy: 1;
+                    };
+                    parseRawLocateValue: typeof parseKimiRawLocateValue;
+                };
+            };
+        };
+    };
+};
+export {};

package/dist/types/ai-model/models/mimo.d.ts

@@ -0,0 +1,10 @@
+import type { ChatCompletionCallContext, ChatCompletionParamsResult } from '../model-adapter/types';
+export declare const mimoAdapters: {
+    'xiaomi-mimo': {
+        chatCompletion: {
+            unsupportedUserConfig: ("reasoningEffort" | "reasoningBudget")[];
+            buildChatCompletionParams: (input: ChatCompletionCallContext) => ChatCompletionParamsResult;
+            useReasoningAsContentFallback: true;
+        };
+    };
+};

package/dist/types/ai-model/models/qwen.d.ts

@@ -0,0 +1,33 @@
+import type { ChatCompletionCallContext, ChatCompletionParamsResult, ModelAdapterDefinition } from '../model-adapter/types';
+import { type LocateResultValue, type PixelBbox } from '../shared/model-locate-result';
+declare function parseQwen25RawLocateValue(input: unknown): LocateResultValue;
+declare function normalizeQwen25ResultToPixelBbox(result: LocateResultValue): PixelBbox;
+export declare const qwenAdapters: {
+    'qwen2.5-vl': {
+        acceptBbox2dAlias: true;
+        chatCompletion: {
+            unsupportedUserConfig: ("reasoningEnabled" | "reasoningEffort" | "reasoningBudget")[];
+            buildChatCompletionParams: (input: ChatCompletionCallContext) => ChatCompletionParamsResult;
+        };
+        imagePreprocess: {
+            padBlockSize: number;
+        };
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: {
+                        readonly shape: "bbox";
+                        readonly order: "xy";
+                    };
+                    parseRawLocateValue: typeof parseQwen25RawLocateValue;
+                    mapLocateResultToPixelBbox: typeof normalizeQwen25ResultToPixelBbox;
+                };
+            };
+        };
+    };
+    'qwen3-vl': ModelAdapterDefinition;
+    qwen3: ModelAdapterDefinition;
+    'qwen3.5': ModelAdapterDefinition;
+    'qwen3.6': ModelAdapterDefinition;
+};
+export {};