@aiscene/core 9.0.6 → 9.0.8

package/dist/types/ai-model/models/registry.d.ts

@@ -0,0 +1,187 @@
+import type { IModelConfig, TModelFamily } from '@aiscene/shared/env';
+import type { ModelAdapter, ModelAdapterDefinition, ModelRuntime } from '../model-adapter/types';
+export declare const MODEL_ADAPTER_CONFIGS: {
+    'xiaomi-mimo': {
+        chatCompletion: {
+            unsupportedUserConfig: ("reasoningEffort" | "reasoningBudget")[];
+            buildChatCompletionParams: (input: import("../model-adapter/types").ChatCompletionCallContext) => import("../model-adapter/types").ChatCompletionParamsResult;
+            useReasoningAsContentFallback: true;
+        };
+    };
+    kimi: {
+        chatCompletion: {
+            unsupportedUserConfig: ("reasoningEffort" | "reasoningBudget")[];
+            buildChatCompletionParams: (input: import("../model-adapter/types").ChatCompletionCallContext) => import("../model-adapter/types").ChatCompletionParamsResult;
+            useReasoningAsContentFallback: true;
+        };
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: {
+                        readonly shape: "point";
+                        readonly order: "xy";
+                        readonly normalizedBy: 1;
+                    };
+                    parseRawLocateValue: (input: unknown) => import("../shared/model-locate-result").LocateResultValue;
+                };
+            };
+        };
+    };
+    kimi3: {
+        chatCompletion: {
+            unsupportedUserConfig: ("reasoningEnabled" | "reasoningBudget")[];
+            buildChatCompletionParams: (input: import("../model-adapter/types").ChatCompletionCallContext) => import("../model-adapter/types").ChatCompletionParamsResult;
+            useReasoningAsContentFallback: true;
+            replayRawAssistantMessage: true;
+        };
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: {
+                        readonly shape: "point";
+                        readonly order: "xy";
+                        readonly normalizedBy: 1;
+                    };
+                    parseRawLocateValue: (input: unknown) => import("../shared/model-locate-result").LocateResultValue;
+                };
+            };
+        };
+    };
+    'gpt-5': {
+        chatCompletion: {
+            unsupportedUserConfig: "reasoningBudget"[];
+            buildChatCompletionParams: (input: import("../model-adapter/types").ChatCompletionCallContext) => import("../model-adapter/types").ChatCompletionParamsResult;
+            resolveImageDetail: (input: import("../model-adapter/types").ChatCompletionCallContext) => import("../model-adapter/types").ImageDetail | undefined;
+        };
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: {
+                        shape: "bbox";
+                        order: "xy";
+                    };
+                };
+            };
+        };
+    };
+    'auto-glm': ModelAdapterDefinition;
+    'auto-glm-multilingual': ModelAdapterDefinition;
+    'glm-v': {
+        chatCompletion: {
+            unsupportedUserConfig: ("reasoningEffort" | "reasoningBudget")[];
+            buildChatCompletionParams: (input: import("../model-adapter/types").ChatCompletionCallContext) => import("../model-adapter/types").ChatCompletionParamsResult;
+            useReasoningAsContentFallback: true;
+        };
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: {
+                        shape: "bbox";
+                        order: "xy";
+                        normalizedBy: number;
+                    };
+                };
+            };
+        };
+    };
+    'vlm-ui-tars': ModelAdapterDefinition;
+    'vlm-ui-tars-doubao': ModelAdapterDefinition;
+    'vlm-ui-tars-doubao-1.5': ModelAdapterDefinition;
+    gemini: {
+        chatCompletion: {
+            unsupportedUserConfig: "reasoningBudget"[];
+            buildChatCompletionParams: (input: import("../model-adapter/types").ChatCompletionCallContext) => import("../model-adapter/types").ChatCompletionParamsResult;
+            messageExtraction: {
+                kind: "custom";
+                extractContentAndReasoning: (message: (import("../model-adapter/types").ChatCompletionContentSource | (Omit<import("openai/resources/index.mjs").ChatCompletionMessage, "content"> & {
+                    content: string | null | (Record<string, unknown> & {
+                        text?: string;
+                        thought?: boolean;
+                    })[];
+                    reasoning_content?: string;
+                    extra_content?: unknown;
+                }) | (Omit<import("openai/resources/index.mjs").ChatCompletionChunk.Choice.Delta, "content"> & {
+                    content: string | null | (Record<string, unknown> & {
+                        text?: string;
+                        thought?: boolean;
+                    })[];
+                    reasoning_content?: string;
+                    extra_content?: unknown;
+                })) | undefined) => import("../model-adapter/types").ContentAndReasoning;
+            };
+        };
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: {
+                        shape: "bbox";
+                        order: "yx";
+                        normalizedBy: number;
+                    };
+                };
+            };
+        };
+    };
+    'doubao-vision': ModelAdapterDefinition;
+    'doubao-seed': ModelAdapterDefinition;
+    deepseek: {
+        chatCompletion: {
+            unsupportedUserConfig: "reasoningBudget"[];
+            buildChatCompletionParams: (input: import("../model-adapter/types").ChatCompletionCallContext) => import("../model-adapter/types").ChatCompletionParamsResult;
+            useReasoningAsContentFallback: true;
+            replayRawAssistantMessage: false;
+        };
+        locate: {
+            element: {
+                protocol: import("../model-adapter/locate-protocol").StandardLocateProtocol;
+                resultFormat: {
+                    coordinates: {
+                        readonly shape: "point";
+                        readonly order: "xy";
+                        readonly normalizedBy: 1000;
+                    };
+                    parseRawLocateValue: (input: unknown) => import("../shared/model-locate-result").LocateResultValue;
+                };
+            };
+            searchArea: {
+                protocol: import("../model-adapter/locate-protocol").StandardLocateProtocol;
+                resultFormat: {
+                    coordinates: {
+                        readonly shape: "bbox";
+                        readonly order: "xy";
+                        readonly normalizedBy: 1000;
+                    };
+                    parseRawLocateValue: (input: unknown) => import("../shared/model-locate-result").LocateResultValue;
+                };
+            };
+        };
+    };
+    'qwen2.5-vl': {
+        acceptBbox2dAlias: true;
+        chatCompletion: {
+            unsupportedUserConfig: ("reasoningEnabled" | "reasoningEffort" | "reasoningBudget")[];
+            buildChatCompletionParams: (input: import("../model-adapter/types").ChatCompletionCallContext) => import("../model-adapter/types").ChatCompletionParamsResult;
+        };
+        imagePreprocess: {
+            padBlockSize: number;
+        };
+        locate: {
+            element: {
+                resultFormat: {
+                    coordinates: {
+                        readonly shape: "bbox";
+                        readonly order: "xy";
+                    };
+                    parseRawLocateValue: (input: unknown) => import("../shared/model-locate-result").LocateResultValue;
+                    mapLocateResultToPixelBbox: (result: import("../shared/model-locate-result").LocateResultValue) => import("../shared/model-locate-result").PixelBbox;
+                };
+            };
+        };
+    };
+    'qwen3-vl': ModelAdapterDefinition;
+    qwen3: ModelAdapterDefinition;
+    'qwen3.5': ModelAdapterDefinition;
+    'qwen3.6': ModelAdapterDefinition;
+};
+export declare function getModelAdapter(modelFamily?: TModelFamily): ModelAdapter;
+export declare function getModelRuntime(config: IModelConfig): ModelRuntime;

package/dist/types/ai-model/models/ui-tars/actions.d.ts

@@ -0,0 +1,3 @@
+import type { PlanningAction } from '../../../types';
+import type { UiTarsParsedPlanningResponse } from './parser';
+export declare function transformUiTarsActions(parsedPlanningResponse: UiTarsParsedPlanningResponse): PlanningAction[];

package/dist/types/ai-model/models/ui-tars/adapter.d.ts

@@ -0,0 +1,6 @@
+import type { ModelAdapterDefinition } from '../../model-adapter/types';
+export declare const uiTarsAdapters: {
+    'vlm-ui-tars': ModelAdapterDefinition;
+    'vlm-ui-tars-doubao': ModelAdapterDefinition;
+    'vlm-ui-tars-doubao-1.5': ModelAdapterDefinition;
+};

package/dist/types/ai-model/models/ui-tars/parser.d.ts

@@ -0,0 +1,72 @@
+import type { Size } from '../../../types';
+import type { UITarsModelVersion } from '@aiscene/shared/env';
+import { actionParser } from '@ui-tars/action-parser';
+type ActionType = 'click' | 'left_double' | 'right_single' | 'drag' | 'type' | 'hotkey' | 'finished' | 'scroll' | 'wait';
+export interface UiTarsParsedPlanningResponse {
+    rawResponse: string;
+    actions: ReturnType<typeof actionParser>['parsed'];
+}
+export declare function parseUiTarsPlanningResponse(rawResponse: string, shotSize: Size, uiTarsModelVersion: UITarsModelVersion): UiTarsParsedPlanningResponse;
+interface BaseAction {
+    action_type: ActionType;
+    action_inputs: Record<string, any>;
+    reflection: string | null;
+    thought: string | null;
+}
+interface ClickAction extends BaseAction {
+    action_type: 'click';
+    action_inputs: {
+        start_box: string;
+    };
+}
+interface DragAction extends BaseAction {
+    action_type: 'drag';
+    action_inputs: {
+        start_box: string;
+        end_box: string;
+    };
+}
+interface WaitAction extends BaseAction {
+    action_type: 'wait';
+    action_inputs: {
+        time: string;
+    };
+}
+interface LeftDoubleAction extends BaseAction {
+    action_type: 'left_double';
+    action_inputs: {
+        start_box: string;
+    };
+}
+interface RightSingleAction extends BaseAction {
+    action_type: 'right_single';
+    action_inputs: {
+        start_box: string;
+    };
+}
+interface TypeAction extends BaseAction {
+    action_type: 'type';
+    action_inputs: {
+        content: string;
+    };
+}
+interface HotkeyAction extends BaseAction {
+    action_type: 'hotkey';
+    action_inputs: {
+        key: string;
+    };
+}
+interface ScrollAction extends BaseAction {
+    action_type: 'scroll';
+    action_inputs: {
+        direction: 'up' | 'down';
+    };
+}
+interface FinishedAction extends BaseAction {
+    action_type: 'finished';
+    action_inputs: {
+        content?: string;
+    };
+}
+export type Action = ClickAction | LeftDoubleAction | RightSingleAction | DragAction | TypeAction | HotkeyAction | ScrollAction | FinishedAction | WaitAction;
+export {};

package/dist/types/ai-model/models/ui-tars/planning.d.ts

@@ -0,0 +1,4 @@
+import type { UITarsModelVersion } from '@aiscene/shared/env';
+import type { CustomPlanningDefinition } from '../../model-adapter/custom-planning-types';
+import { type UiTarsParsedPlanningResponse } from './parser';
+export declare function createUiTarsPlanner(uiTarsModelVersion: UITarsModelVersion): CustomPlanningDefinition<UiTarsParsedPlanningResponse>;

package/dist/types/ai-model/models/ui-tars/prompt.d.ts

@@ -0,0 +1,2 @@
+export declare function getUiTarsPlanningPrompt(): string;
+export declare const getSummary: (prediction: string) => string;

package/dist/types/ai-model/models/utils/intent.d.ts

@@ -0,0 +1,2 @@
+import type { TIntent } from '@aiscene/shared/env';
+export declare function isLocateIntent(intent?: TIntent): boolean;

package/dist/types/ai-model/prompt/insight/index.d.ts

@@ -0,0 +1,2 @@
+export { buildInsightSystemPrompt } from './system-prompt';
+export { buildTypeQueryDemandValue, extractDataQueryPrompt, } from './user-prompt';

package/dist/types/ai-model/prompt/insight/system-prompt.d.ts

@@ -0,0 +1,6 @@
+import type { InsightProtocol } from '../../model-adapter/insight-protocol';
+export declare function buildInsightSystemPrompt(options: {
+    screenshotIncluded?: boolean;
+    referenceImagesIncluded?: boolean;
+    insightProtocol: InsightProtocol;
+}): string;

package/dist/types/ai-model/prompt/insight/user-prompt.d.ts

@@ -0,0 +1,3 @@
+import type { ServiceExtractParam } from '../../../types';
+export declare function buildTypeQueryDemandValue(type: 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor', demand: ServiceExtractParam): string;
+export declare const extractDataQueryPrompt: (pageDescription: string, dataQuery: string | Record<string, string>, context?: string) => string;

package/dist/types/ai-model/prompt/locate/element-system-prompt.d.ts

@@ -0,0 +1,4 @@
+export declare function buildElementLocateSystemPrompt({ systemPromptIntroduction, responseInstructions, }: {
+    systemPromptIntroduction: string;
+    responseInstructions: string;
+}): string;

package/dist/types/ai-model/prompt/locate/grounding-rules.d.ts

@@ -0,0 +1 @@
+export declare function locateGroundingRules(): string;

package/dist/types/ai-model/prompt/locate/index.d.ts

@@ -0,0 +1,3 @@
+export { buildElementLocateSystemPrompt } from './element-system-prompt';
+export { locateGroundingRules } from './grounding-rules';
+export { buildSearchAreaLocateSystemPrompt } from './search-area-system-prompt';

package/dist/types/ai-model/prompt/locate/search-area-system-prompt.d.ts

@@ -0,0 +1,4 @@
+export declare function buildSearchAreaLocateSystemPrompt({ systemPromptIntroduction, responseInstructions, }: {
+    systemPromptIntroduction: string;
+    responseInstructions: string;
+}): string;

package/dist/types/ai-model/prompt/planning/action-output-example.d.ts

@@ -0,0 +1,17 @@
+import type { DeviceAction } from '../../../types';
+import type { PlanningActionOutputProtocol } from '../../model-adapter/planning-protocol';
+import type { LocateResultPromptSpec } from '../../shared/model-locate-result';
+export type ActionOutputExampleDefinition = Pick<DeviceAction<any>, 'name' | 'paramSchema' | 'sample'>;
+export declare const buildActionOutputExample: (action: ActionOutputExampleDefinition, { locatePromptSpec, locateResultExampleIndex, buildActionOutput, }: {
+    locatePromptSpec?: LocateResultPromptSpec;
+    /**
+     * Selects the locate result example value injected into this action.
+     * Keep the same index for the same example across multi-turn prompt
+     * variants, and use different indexes for different examples to avoid
+     * reusing identical locate coordinates.
+     */
+    locateResultExampleIndex?: number;
+    buildActionOutput: PlanningActionOutputProtocol["buildActionOutput"];
+}) => string | undefined;
+export declare const createSampleTapAction: (prompt: string) => ActionOutputExampleDefinition;
+export declare const createSampleInputAction: (value: string) => ActionOutputExampleDefinition;

package/dist/types/ai-model/prompt/planning/action-space-description.d.ts

@@ -0,0 +1,11 @@
+import type { DeviceAction } from '../../../types';
+import type { PlanningActionSpaceFormat, StandardPlanningProtocol } from '../../model-adapter/planning-protocol';
+import type { LocateResultPromptSpec } from '../../shared/model-locate-result';
+type BuildPlanningActionSpaceDescriptionInput = {
+    actionSpace: DeviceAction<any>[];
+    locatePromptSpec?: LocateResultPromptSpec;
+    planningProtocol: StandardPlanningProtocol;
+};
+export declare const serializeActionDescriptions: (actionDescriptions: unknown[], format: PlanningActionSpaceFormat) => string;
+export declare const buildPlanningActionSpaceDescription: ({ actionSpace, locatePromptSpec, planningProtocol, }: BuildPlanningActionSpaceDescriptionInput) => string;
+export {};

package/dist/types/ai-model/prompt/planning/index.d.ts

@@ -0,0 +1,6 @@
+export { buildStandardPlanningSystemPrompt } from './system-prompt';
+export { buildPlanningActionSpaceDescription, serializeActionDescriptions, } from './action-space-description';
+export { buildActionOutputExample, createSampleInputAction, createSampleTapAction, } from './action-output-example';
+export type { ActionOutputExampleDefinition } from './action-output-example';
+export { buildPlanningResponseExample } from './planning-response-example';
+export { buildSubGoalsText } from './sub-goals-text';

package/dist/types/ai-model/prompt/planning/multi-turn-example.d.ts

@@ -0,0 +1,10 @@
+import type { PlanningActionOutputProtocol } from '../../model-adapter/planning-protocol';
+import type { LocateResultPromptSpec } from '../../shared/model-locate-result';
+export declare const buildPlanningMultiTurnExample: ({ includeSubGoals, includeThought, includeLog, locatePromptSpec, actionOutputProtocol, prefix, }: {
+    includeSubGoals: boolean;
+    includeThought: boolean;
+    includeLog: boolean;
+    locatePromptSpec?: LocateResultPromptSpec;
+    actionOutputProtocol: PlanningActionOutputProtocol;
+    prefix?: string;
+}) => string;

package/dist/types/ai-model/prompt/planning/planning-response-example.d.ts

@@ -0,0 +1,28 @@
+import type { SubGoalStatus } from '../../../types';
+type PlanningSubGoalExample = {
+    index: number;
+    status: Extract<SubGoalStatus, 'pending' | 'finished'>;
+    description: string;
+};
+type PlanningResponseActionOutputExample = {
+    log?: string;
+    actionOutputExample: string | undefined;
+    complete?: never;
+};
+type PlanningResponseCompleteExample = {
+    complete: {
+        success: boolean;
+        message: string;
+    };
+    log?: never;
+    actionOutputExample?: never;
+};
+export type BuildPlanningResponseExampleInput = {
+    prefix?: string;
+    planning?: string;
+    updateSubGoals?: PlanningSubGoalExample[];
+    markSubGoalsDone?: number[];
+    memory?: string;
+} & (PlanningResponseActionOutputExample | PlanningResponseCompleteExample);
+export declare const buildPlanningResponseExample: (input: BuildPlanningResponseExampleInput) => string;
+export {};

package/dist/types/ai-model/prompt/planning/sub-goals-text.d.ts

@@ -0,0 +1,2 @@
+import type { SubGoal } from '../../../types';
+export declare const buildSubGoalsText: (subGoals: readonly SubGoal[]) => string;

package/dist/types/ai-model/prompt/planning/system-prompt.d.ts

@@ -0,0 +1,18 @@
+import type { DeviceAction } from '../../../types';
+import type { StandardPlanningProtocol } from '../../model-adapter/planning-protocol';
+import type { LocateResultPromptSpec } from '../../shared/model-locate-result';
+type BuildStandardPlanningSystemPromptInput = {
+    actionSpace: DeviceAction<any>[];
+    includeSubGoals?: boolean;
+    includeThought?: boolean;
+    includeLog?: boolean;
+    planningProtocol: StandardPlanningProtocol;
+} & ({
+    includeLocateInPlanning: true;
+    locatePromptSpec: LocateResultPromptSpec;
+} | {
+    includeLocateInPlanning: false;
+    locatePromptSpec?: never;
+});
+export declare function buildStandardPlanningSystemPrompt(input: BuildStandardPlanningSystemPromptInput): Promise<string>;
+export {};

package/dist/types/ai-model/service-caller/index.d.ts

@@ -3,39 +3,53 @@ import type { StreamingCallback } from '../../types';
 export declare class AIResponseParseError extends Error {
     usage?: AIUsageInfo;
     rawResponse: string;
-    constructor(message: string, rawResponse: string, usage?: AIUsageInfo);
+    rawChoiceMessage?: unknown;
+    reasoningContent?: string;
+    constructor(message: string, rawResponse: string, usage?: AIUsageInfo, rawChoiceMessage?: unknown, reasoningContent?: string);
 }
-import { type IModelConfig, type TModelFamily } from '@aiscene/shared/env';
+import { type IModelConfig, type TModelFamily, type TModelReasoningEnabled } from '@aiscene/shared/env';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
 import type { AIArgs } from '../../common';
-export declare function callAI(messages: ChatCompletionMessageParam[], modelConfig: IModelConfig, options?: {
+import type { ModelRuntime } from '../models';
+type ModelRuntimeInput = IModelConfig | ModelRuntime;
+export declare function callAI(messages: ChatCompletionMessageParam[], modelRuntimeOrConfig: ModelRuntimeInput, options?: {
     stream?: boolean;
     onChunk?: StreamingCallback;
     abortSignal?: AbortSignal;
+    requiresOriginalImageDetail?: boolean;
+    expectedJsonObjectResponse?: boolean;
+    semanticRetryAttempt?: number;
 }): Promise<{
     content: string;
     reasoning_content?: string;
+    rawChoiceMessage?: unknown;
     usage?: AIUsageInfo;
     isStreamed: boolean;
 }>;
-export declare function callAIWithObjectResponse<T>(messages: ChatCompletionMessageParam[], modelConfig: IModelConfig, options?: {
+export declare function callAIWithObjectResponse<T>(messages: ChatCompletionMessageParam[], modelRuntimeOrConfig: ModelRuntimeInput, options?: {
     abortSignal?: AbortSignal;
+    jsonParserSource?: unknown;
+    retryTimes?: number;
+    retryInterval?: number;
 }): Promise<{
     content: T;
     contentString: string;
     usage?: AIUsageInfo;
     reasoning_content?: string;
+    rawChoiceMessage?: unknown;
 }>;
-export declare function callAIWithStringResponse(msgs: AIArgs, modelConfig: IModelConfig, options?: {
+export declare function callAIWithStringResponse(msgs: AIArgs, modelRuntimeOrConfig: ModelRuntimeInput, options?: {
     abortSignal?: AbortSignal;
+    requiresOriginalImageDetail?: boolean;
 }): Promise<{
     content: string;
     usage?: AIUsageInfo;
+    rawChoiceMessage?: unknown;
 }>;
 export declare function extractJSONFromCodeBlock(response: string): string;
 export declare function preprocessDoubaoBboxJson(input: string): string;
 export declare function resolveReasoningConfig({ reasoningEnabled, reasoningEffort, reasoningBudget, modelFamily, }: {
-    reasoningEnabled?: boolean;
+    reasoningEnabled?: TModelReasoningEnabled;
     reasoningEffort?: string;
     reasoningBudget?: number;
     modelFamily?: TModelFamily;
@@ -44,3 +58,4 @@ export declare function resolveReasoningConfig({ reasoningEnabled, reasoningEffo
     debugMessage?: string;
 };
 export declare function safeParseJson(input: string, modelFamily: TModelFamily | undefined): any;
+export {};

package/dist/types/ai-model/service-caller/model-call-recorder.d.ts

@@ -0,0 +1,12 @@
+export declare function isModelCallRecordingEnabled(): boolean;
+export declare class ModelCallRecorder {
+    private readonly processStartTime;
+    private filePath;
+    private filePathPromise;
+    private writeQueue;
+    private initializeFilePath;
+    private getFilePath;
+    private writeEvent;
+    record(event: Record<string, unknown>): Promise<void>;
+}
+export declare function recordModelCallEvent(event: Record<string, unknown>): Promise<void>;

package/dist/types/ai-model/service-caller/openai-error.d.ts

@@ -0,0 +1,26 @@
+export interface OpenAIErrorResponseContext {
+    recordEvent?: (event: Record<string, unknown>) => void;
+    responseRequestIds?: Array<{
+        attempt: number;
+        requestId: string;
+        status: number;
+        ok: boolean;
+    }>;
+    rawResponseBodies?: Array<{
+        attempt: number;
+        body: string;
+    }>;
+    fetchErrors?: Array<{
+        attempt: number;
+        error: string;
+    }>;
+    httpResponses?: Array<{
+        attempt: number;
+        status: number;
+        ok: boolean;
+        headers: Array<[string, string]>;
+        body?: string;
+    }>;
+}
+export declare function wrapOpenAICompatibleFetch(context: OpenAIErrorResponseContext): typeof fetch;
+export declare function formatOpenAIAPIErrorDetails(_error: unknown, context: OpenAIErrorResponseContext): string;

package/dist/types/ai-model/service-caller/proxy-deps.node.d.ts

@@ -0,0 +1,2 @@
+export declare function loadUndici(): Promise<typeof import('undici')>;
+export declare function loadFetchSocks(): Promise<typeof import('fetch-socks')>;

package/dist/types/ai-model/service-caller/proxy-deps.stub.d.ts

@@ -0,0 +1,2 @@
+export declare function loadUndici(): Promise<typeof import('undici')>;
+export declare function loadFetchSocks(): Promise<typeof import('fetch-socks')>;

package/dist/types/ai-model/service-caller/semantic-retry.d.ts

@@ -0,0 +1,30 @@
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+export declare function withSemanticRetryFeedback(messages: ChatCompletionMessageParam[], previousParseError?: unknown): ChatCompletionMessageParam[];
+export type CallAiAndParseWithRetryOptions<Response, Parsed> = {
+    /** Sends a single model request. Request errors are always propagated as-is. */
+    /**
+     * `retryAttempt` is local to this retry loop: 0 for the initial request,
+     * then incremented after each parsing failure.
+     */
+    callAi: (retryAttempt: number, previousParseError?: unknown) => Promise<Response>;
+    /** Parses a successful model response. Only failures from this callback retry. */
+    parseResponse: (response: Response) => Parsed | Promise<Parsed>;
+    /** Converts the final parsing failure into the caller's domain error. */
+    toParseError: (error: unknown, response: Response) => Error;
+    /** Number of additional model requests after a parsing failure. Defaults to 0. */
+    parseRetryTimes?: number;
+    /** Delay in milliseconds before retrying a parsing failure. Defaults to 0. */
+    parseRetryInterval?: number;
+    abortSignal?: AbortSignal;
+    /** Runs immediately before a parsing failure triggers another model request. */
+    onParseRetry?: (error: unknown, response: Response) => void;
+};
+/**
+ * Calls AI and parses its response, retrying only when the response cannot be
+ * structurally parsed.
+ *
+ * Errors thrown by callAi() intentionally bypass this retry mechanism and are
+ * propagated as-is. Only a successful callAi() response that fails the
+ * parseResponse() callback triggers another model request.
+ */
+export declare function callAiAndParseWithRetry<Response, Parsed>({ callAi, parseResponse, toParseError, parseRetryTimes, parseRetryInterval, abortSignal, onParseRetry, }: CallAiAndParseWithRetryOptions<Response, Parsed>): Promise<Parsed>;

package/dist/types/ai-model/service-caller/types.d.ts

@@ -0,0 +1,2 @@
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+export type AIArgs = ChatCompletionMessageParam[];

package/dist/types/ai-model/shared/action-schema.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Find ZodDefault in the wrapper chain and return its default value.
+ */
+export declare const getZodDefaultValue: (field: unknown) => unknown | undefined;

package/dist/types/ai-model/shared/json.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Extract the JSON portion from a model response.
+ *
+ * This mainly handles fenced JSON like ```json { ... } ``` by removing the
+ * fence. If that does not produce a JSON object, it falls back to a greedy regex
+ * that extracts from the first "{" to the last "}". If that still fails, the
+ * original response is returned.
+ *
+ * Expected model responses are JSON objects, possibly wrapped in markdown
+ * fences. Natural language mixed with JSON, or arrays like
+ * [{"type":"Tap"}, {"type":"Hover"}], are outside the supported contract and
+ * are not reliably recoverable.
+ *
+ * This extractor is also used by extraction responses that can be any
+ * JSON value. In those cases, arrays/strings/numbers have no object braces and
+ * pass through unchanged.
+ */
+export declare function extractJSONFromCodeBlock(response: string): string;
+export type JsonParserSource = 'generic-object' | 'planning-action-param' | 'locate' | 'section-locator';
+export interface JsonParserContext {
+    source: JsonParserSource;
+    preserveStringValueKeys?: string[];
+}
+export type JsonParser = (raw: string, context?: JsonParserContext) => unknown;
+export declare function assertJsonObject(parsed: unknown): asserts parsed is Record<string, unknown>;
+export declare function parseModelResponseJson(raw: string, context?: JsonParserContext): any;

package/dist/types/ai-model/shared/model-locate-result/bbox.d.ts

@@ -0,0 +1,6 @@
+import type { LocateResultBbox, LocateResultContext, PixelBbox } from './types';
+export declare function maxPixelIndex(size: number): number;
+export declare function normalizedCoordinateToPixelIndex(value: number, normalizedBy: number, size: number): number;
+export declare function mapNormalizedCoordinatesToPixelBbox(coordinates: LocateResultBbox, normalizedBy: number, width: number, height: number): PixelBbox;
+export declare function expandPointToBbox(x: number, y: number, maxX: number, maxY: number, halfSize: number): LocateResultBbox;
+export declare function finalizePixelBbox(pixelBbox: PixelBbox, rawResult: unknown, { preparedSize, contentSize }: LocateResultContext): PixelBbox;

package/dist/types/ai-model/shared/model-locate-result/coordinate-distance.d.ts

@@ -0,0 +1,4 @@
+import type { Size } from '../../../types';
+import type { ResolvedLocateResultCoordinates } from './types';
+export type CoordinateDistanceAxis = 'x' | 'y';
+export declare function createCoordinateDistanceToPixels(size: Size, coordinateSystem: ResolvedLocateResultCoordinates): (delta: number, axis: CoordinateDistanceAxis) => number;

package/dist/types/ai-model/shared/model-locate-result/errors.d.ts

@@ -0,0 +1,2 @@
+export declare const defaultModelFamilyRequiredForLocateMessage = "Default model family is required for locate. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse locate coordinates correctly. https://midscenejs.com/model-config";
+export declare function planningModelFamilyRequiredForLocateMessage(slot?: string): "Planning model family is required because aiAct is asking the planning model to return locate coordinates. Configure MIDSCENE_PLANNING_MODEL_FAMILY for the planning model, or remove the separate planning model config and configure MIDSCENE_MODEL_FAMILY on the default model. https://midscenejs.com/model-config" | "Default model family is required because aiAct is asking the default model to return locate coordinates during planning. Configure MIDSCENE_MODEL_FAMILY so Midscene can parse planning locate coordinates correctly. https://midscenejs.com/model-config";

package/dist/types/ai-model/shared/model-locate-result/factory.d.ts

@@ -0,0 +1,3 @@
+import type { LocateResultCodec, LocateResultCoordinates, LocateResultFormatDefinition, ResolvedLocateResultCoordinates } from './types';
+export declare function resolveLocateResultCoordinates(coordinates: LocateResultCoordinates): ResolvedLocateResultCoordinates;
+export declare function createLocateResultCodec(config: LocateResultFormatDefinition): LocateResultCodec;