@aiscene/core 9.0.6 → 9.0.8

package/dist/types/ai-model/shared/model-locate-result/index.d.ts

@@ -0,0 +1,7 @@
+export { createLocateResultCodec, resolveLocateResultCoordinates, } from './factory';
+export { formatLocateExampleValue } from './prompt-spec';
+export { createCoordinateDistanceToPixels } from './coordinate-distance';
+export { isBboxLocateResultValue, isPointLocateResultValue, } from './types';
+export type { CoordinateDistanceAxis } from './coordinate-distance';
+export { createLocateResultValue, parseCoordinateList, unwrapCoordinateListLikeInput, } from './parse';
+export type { LocateResultBbox, PixelBbox, RawLocateValue, LocateResultCodec, LocateResultFormatDefinition, LocateResultCoordinates, LocateResultContext, LocateResultShape, LocateResultPromptSpec, LocateResultValue, NonEmptyArray, ResolvedLocateResultCoordinates, } from './types';

package/dist/types/ai-model/shared/model-locate-result/parse.d.ts

@@ -0,0 +1,8 @@
+import type { BboxLocateResultCoordinates, BboxLocateResultValue, LocateResultValue, PointLocateResultCoordinates, PointLocateResultValue, ResolvedLocateResultCoordinates } from './types';
+type CoordinateListLikeInput = number[] | string[] | string | (number[] | string[])[];
+export declare function unwrapCoordinateListLikeInput(coordinateList: CoordinateListLikeInput): number[] | string[] | string;
+export declare function parseCoordinateList(input: unknown, label: string): number[];
+export declare function createLocateResultValue(coordinatesMeta: PointLocateResultCoordinates, coordinates: number[]): PointLocateResultValue;
+export declare function createLocateResultValue(coordinatesMeta: BboxLocateResultCoordinates, coordinates: number[]): BboxLocateResultValue;
+export declare function parseNumericLocateResult(resolvedCoordinates: ResolvedLocateResultCoordinates, input: unknown): LocateResultValue;
+export {};

package/dist/types/ai-model/shared/model-locate-result/pixel-bbox-mapper.d.ts

@@ -0,0 +1,7 @@
+import type { LocateResultValue, PixelBbox } from './types';
+export declare function mapLocateResultToPixelBboxByCoordinates(result: LocateResultValue, { preparedSize }: {
+    preparedSize: {
+        width: number;
+        height: number;
+    };
+}): PixelBbox;

package/dist/types/ai-model/shared/model-locate-result/prompt-spec.d.ts

@@ -0,0 +1,6 @@
+import type { LocateResultBbox, LocateResultPromptSpec, ResolvedLocateResultCoordinates } from './types';
+export declare function formatLocateExampleValue(value: unknown): string;
+export declare function describeLocateResultValueSchema({ shape, }: ResolvedLocateResultCoordinates): string;
+export declare function locateResultExampleValue(resolvedCoordinates: ResolvedLocateResultCoordinates, region: LocateResultBbox): number[];
+export declare const locateResultExampleRegions: LocateResultBbox[];
+export declare function createLocateResultPromptSpec(resolvedCoordinates: ResolvedLocateResultCoordinates): LocateResultPromptSpec;

package/dist/types/ai-model/shared/model-locate-result/types.d.ts

@@ -0,0 +1,150 @@
+import type { Bbox } from '../../../types';
+export type { Bbox };
+export type LocateResultBbox = Bbox;
+export type PixelBbox = Bbox;
+export type NonEmptyArray<T> = [T, ...T[]];
+export type RawLocateValue = unknown;
+export type LocateResultPoint = [number, number];
+export type PointLocateResultCoordinates = ResolvedLocateResultCoordinates & {
+    shape: 'point';
+};
+export type BboxLocateResultCoordinates = ResolvedLocateResultCoordinates & {
+    shape: 'bbox';
+};
+export type LocateResultValue = {
+    coordinates: LocateResultPoint;
+    coordinatesMeta: PointLocateResultCoordinates;
+} | {
+    coordinates: LocateResultBbox;
+    coordinatesMeta: BboxLocateResultCoordinates;
+};
+export type PointLocateResultValue = Extract<LocateResultValue, {
+    coordinatesMeta: {
+        shape: 'point';
+    };
+}>;
+export type BboxLocateResultValue = Extract<LocateResultValue, {
+    coordinatesMeta: {
+        shape: 'bbox';
+    };
+}>;
+export declare function isBboxLocateResultValue(result: LocateResultValue): result is BboxLocateResultValue;
+export declare function isPointLocateResultValue(result: LocateResultValue): result is PointLocateResultValue;
+export type LocateResultShape = 'bbox' | 'point';
+export interface LocateResultContext {
+    preparedSize: {
+        width: number;
+        height: number;
+    };
+    contentSize?: {
+        width: number;
+        height: number;
+    };
+}
+export interface LocateResultPromptSpec {
+    resultKey: string;
+    resultValueSchema: string;
+    resultValueDescription: string;
+    resultNoun: string;
+    resultNounPlural: string;
+    exampleValues: NonEmptyArray<unknown>;
+}
+export interface LocateResultCodec {
+    promptSpec: LocateResultPromptSpec;
+    toPixelBbox(input: RawLocateValue, ctx: LocateResultContext): PixelBbox;
+}
+export interface LocateResultCoordinates {
+    shape: LocateResultShape;
+    order?: 'xy' | 'yx';
+    normalizedBy?: number;
+}
+export type ResolvedLocateResultCoordinates = {
+    shape: 'point';
+    order: 'xy' | 'yx';
+    normalizedBy?: number;
+} | {
+    shape: 'bbox';
+    order: 'xy' | 'yx';
+    normalizedBy?: number;
+};
+export type RawLocateValueParser = (input: RawLocateValue) => LocateResultValue;
+export type LocateResultPixelBboxMapper = (result: LocateResultValue, ctx: LocateResultContext) => PixelBbox;
+/**
+ * Declarative config for the standard locate workflow.
+ *
+ * The standard workflow has three steps:
+ * 1. `coordinates` is expanded into prompt wording, a default
+ *    raw result parser, and a default pixel bbox mapper.
+ * 2. `parseRawLocateValue` converts that raw result value into Midscene's
+ *    internal `LocateResultValue` shape:
+ *    `{ coordinates, coordinatesMeta }`. Omit it when the model returns a
+ *    plain numeric bbox/point matching `coordinates`; provide it when the
+ *    model needs repair, fallback handling, or per-result coordinate metadata.
+ * 3. `mapLocateResultToPixelBbox` converts the parsed result into a pixel bbox
+ *    `[left, top, right, bottom]`. Omit it when `coordinates` is enough to describe
+ *    the coordinate system and order; provide it only for model-specific
+ *    conversion rules.
+ *
+ * This format only describes one raw coordinate value. The operation protocol
+ * owns the response envelope and identifies target/reference values before the
+ * runtime codec converts each value independently.
+ *
+ * Example 1: a GLM-like model that directly matches the standard coordinates.
+ *
+ * ```ts
+ * resultFormat: {
+ *   coordinates: { shape: 'bbox', order: 'xy', normalizedBy: 1000 },
+ * }
+ * ```
+ *
+ * Example 2: Qwen 2.5 returns pixel coordinates, but may return a point-like
+ * value that needs custom parsing/fallback. The default pixel bbox mapper is
+ * bypassed only if custom fallback sizing is required.
+ *
+ * ```ts
+ * resultFormat: {
+ *   coordinates: { shape: 'bbox', order: 'xy' },
+ *   parseRawLocateValue: parseQwen25RawLocateValue,
+ *   mapLocateResultToPixelBbox: normalizeQwen25ResultToPixelBbox,
+ * }
+ * ```
+ *
+ * Example 3: a model with a custom raw value shape can keep the standard
+ * workflow while replacing parsing and mapping.
+ *
+ * ```ts
+ * resultFormat: {
+ *   coordinates: { shape: 'bbox', order: 'xy' },
+ *   parseRawLocateValue: (raw) => ({
+ *     coordinates: [
+ *       Number((raw as any).left),
+ *       Number((raw as any).top),
+ *       Number((raw as any).right),
+ *       Number((raw as any).bottom),
+ *     ],
+ *     coordinatesMeta: { shape: 'bbox', order: 'xy' },
+ *   }),
+ *   mapLocateResultToPixelBbox: (result) => result.coordinates,
+ * }
+ * ```
+ */
+export type LocateResultFormatDefinition = {
+    /**
+     * Common locate result coordinates shorthand. This is the preferred config surface
+     * for normal models because it keeps result type, coordinate system, and
+     * coordinate order in one orthogonal field.
+     */
+    coordinates: LocateResultCoordinates;
+    /**
+     * Parses one raw coordinate value into a `LocateResultValue`. This function
+     * should handle response repair, bbox-vs-point fallback, and the coordinate
+     * metadata that describes the parsed coordinates.
+     */
+    parseRawLocateValue?: RawLocateValueParser;
+    /**
+     * Maps the parsed result into a pixel bbox. Most models should omit this
+     * and let `coordinates` drive the default conversion. Provide it only when point
+     * fallback size, clipping, or coordinate semantics are model-specific.
+     */
+    mapLocateResultToPixelBbox?: LocateResultPixelBboxMapper;
+};

package/dist/types/ai-model/shared/multimodal-prompt.d.ts

@@ -0,0 +1,13 @@
+import { type TMultimodalPrompt, type TUserPrompt } from '../../common';
+import type { ChatCompletionUserMessageParam } from 'openai/resources/index';
+export interface PreparedReferenceImage {
+    name: string;
+    url: string;
+}
+export interface PreparedUserPrompt {
+    text: string;
+    referenceImages: PreparedReferenceImage[];
+}
+export declare const prepareUserPrompt: (userPrompt: TUserPrompt) => Promise<PreparedUserPrompt>;
+export declare const preparedReferenceImagesToChatMessages: (referenceImages: PreparedReferenceImage[]) => ChatCompletionUserMessageParam[];
+export declare const multimodalPromptToChatMessages: (multimodalPrompt?: TMultimodalPrompt) => Promise<ChatCompletionUserMessageParam[]>;

package/dist/types/ai-model/shared/xml.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Extract content from an XML tag in a string, searching from the end.
+ * This approach handles cases where models prepend thinking content (like <think>...</think>)
+ * before the actual response tags, or when there are incomplete/nested tags.
+ *
+ * Strategy: Find the LAST closing tag, then search backwards for the nearest opening tag.
+ * This ensures we get the last complete tag pair, even if there are incomplete tags before it.
+ *
+ * @param xmlString - The XML string to parse
+ * @param tagName - The name of the tag to extract (case-insensitive)
+ * @returns The trimmed content of the tag, or undefined if not found
+ */
+export declare function extractXMLTag(xmlString: string, tagName: string): string | undefined;
+/** Extract raw XML from the first protocol tag through the last protocol tag. */
+export declare function extractRawXMLFragment(content: string, tagNames: readonly [string, ...string[]]): string;

package/dist/types/ai-model/workflows/grounding/index.d.ts

@@ -0,0 +1,3 @@
+export { AiLocateElement } from './locate';
+export { AiLocateSection } from './section-locate';
+export { buildSearchAreaConfig } from './search-area';

package/dist/types/ai-model/workflows/grounding/locate-result-rect.d.ts

@@ -0,0 +1,5 @@
+import type { PixelBbox } from '../../shared/model-locate-result';
+import type { Rect } from '../../../types';
+/** Merge model pixel bboxes into one screenshot rectangle. */
+export declare function mergePixelBboxesToRect(pixelBboxes: PixelBbox[]): Rect;
+export declare function pixelBboxToRect([left, top, right, bottom]: PixelBbox): Rect;

package/dist/types/ai-model/workflows/grounding/locate.d.ts

@@ -0,0 +1,7 @@
+import type { TUserPrompt } from '../../../common';
+import type { LocateModelResponse, LocateOptions, LocateRequest, LocateResult } from './types';
+export { userPromptToString as extraTextFromUserPrompt } from '../../../common';
+export declare function AiLocateElement(options: LocateOptions & {
+    targetElementDescription: TUserPrompt;
+}): Promise<LocateResult>;
+export declare function genericLocate(locateRequest: LocateRequest): Promise<LocateModelResponse>;

package/dist/types/ai-model/workflows/grounding/planning-action-locate.d.ts

@@ -0,0 +1,4 @@
+import type { ResolvedCustomPlanningDefinition } from '../../model-adapter/custom-planning-types';
+import type { PlanningTapLocatorDefinition } from '../../model-adapter/types';
+import type { LocateFn } from './types';
+export declare function resolvePlanningTapLocator<TParsed>(definition: PlanningTapLocatorDefinition, planner: ResolvedCustomPlanningDefinition<TParsed>): LocateFn;

package/dist/types/ai-model/workflows/grounding/search-area-mapping.d.ts

@@ -0,0 +1,4 @@
+import type { PixelBbox } from '../../shared/model-locate-result';
+import type { SearchAreaImageMapping } from './types';
+/** Map a bbox in the prepared search-area image back to screenshot pixels. */
+export declare function mapSearchAreaPixelBboxToOriginalPixelBbox([left, top, right, bottom]: PixelBbox, mapping?: SearchAreaImageMapping): PixelBbox;

package/dist/types/ai-model/workflows/grounding/search-area.d.ts

@@ -0,0 +1,18 @@
+import type { Rect, Size, UIContext } from '../../../types';
+import type { SearchAreaConfig } from './types';
+/**
+ * Expand the search area to at least 400 x 400 pixels
+ *
+ * Step 1: Extend 100px on each side (top, right, bottom, left)
+ * - If the element is near a boundary, expansion on that side will be limited
+ * - No compensation is made for boundary limitations (this is intentional)
+ *
+ * Step 2: Ensure the area is at least 400x400 pixels
+ * - Scale up proportionally from the center if needed
+ * - Final result is clamped to screen boundaries
+ */
+export declare function expandSearchArea(rect: Rect, screenSize: Size): Rect;
+export declare function buildSearchAreaConfig(options: {
+    context: UIContext;
+    baseRect: Rect;
+}): Promise<SearchAreaConfig>;

package/dist/types/ai-model/workflows/grounding/section-locate.d.ts

@@ -0,0 +1,16 @@
+import type { AIUsageInfo, UIContext } from '../../../types';
+import type { TUserPrompt } from '../../../common';
+import type { ModelRuntime } from '../../models';
+import type { SearchAreaConfig } from './types';
+export declare function AiLocateSection(options: {
+    context: UIContext;
+    sectionDescription: TUserPrompt;
+    modelRuntime: ModelRuntime;
+    abortSignal?: AbortSignal;
+}): Promise<{
+    searchAreaConfig?: SearchAreaConfig;
+    error?: string;
+    rawResponse: string;
+    rawChoiceMessage?: unknown;
+    usage?: AIUsageInfo;
+}>;

package/dist/types/ai-model/workflows/grounding/types.d.ts

@@ -0,0 +1,56 @@
+import type { AIUsageInfo, PixelBbox, Rect, UIContext } from '../../../types';
+import type { LocateResultElement } from '@aiscene/shared/types';
+import type { TUserPrompt } from '../../../common';
+import type { ModelRuntime } from '../../models';
+/** Maps coordinates from a prepared search-area image back to the screenshot. */
+export interface SearchAreaImageMapping {
+    offset: {
+        x: number;
+        y: number;
+    };
+    scale: number;
+}
+export interface SearchAreaConfig {
+    sourceRect: Rect;
+    image: {
+        imageBase64: string;
+        width: number;
+        height: number;
+    };
+    mapping: SearchAreaImageMapping;
+}
+export interface LocateOptions {
+    context: UIContext;
+    searchConfig?: SearchAreaConfig;
+    modelRuntime: ModelRuntime;
+    abortSignal?: AbortSignal;
+}
+export interface LocateRequest {
+    targetElementDescription: TUserPrompt;
+    locateImage: {
+        imageBase64: string;
+        width: number;
+        height: number;
+    };
+    options: LocateOptions;
+}
+export interface LocateResult {
+    parseResult: {
+        element?: LocateResultElement;
+        errors?: string[];
+    };
+    rect?: Rect;
+    rawResponse: string;
+    rawChoiceMessage?: unknown;
+    usage?: AIUsageInfo;
+    reasoning_content?: string;
+}
+export interface LocateModelResponse {
+    locatedPixelBbox?: PixelBbox;
+    rawResponse: string;
+    rawChoiceMessage?: unknown;
+    usage?: AIUsageInfo;
+    reasoningContent?: string;
+    errors?: string[];
+}
+export type LocateFn = (locateRequest: LocateRequest) => Promise<LocateModelResponse>;

package/dist/types/ai-model/workflows/grounding/utils.d.ts

@@ -0,0 +1,25 @@
+import { type TUserPrompt } from '../../../common';
+import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
+import { type ImagePreprocessPolicy, type PreparedModelImage } from '../../model-adapter/image-preprocess';
+import type { LocateUserMessageContentOrder } from '../../model-adapter/types';
+import type { ModelRuntime } from '../../models';
+export type GroundingAIArgs = [
+    ChatCompletionSystemMessageParam,
+    ...ChatCompletionUserMessageParam[]
+];
+export declare function prepareLocateModelInput({ locateImage, targetDescription, systemPrompt, userPrompt, imagePreprocess, userMessageContentOrder, }: {
+    locateImage: {
+        imageBase64: string;
+        width: number;
+        height: number;
+    };
+    targetDescription: TUserPrompt;
+    systemPrompt: string;
+    userPrompt: string;
+    imagePreprocess: ImagePreprocessPolicy;
+    userMessageContentOrder: LocateUserMessageContentOrder;
+}): Promise<{
+    messages: GroundingAIArgs;
+    preparedImage: PreparedModelImage;
+}>;
+export declare function formatLocateModelContext(modelRuntime: ModelRuntime): string;

package/dist/types/ai-model/workflows/insight/extraction.d.ts

@@ -0,0 +1,18 @@
+import type { ServiceExtractOption, UIContext } from '../../../types';
+import type { TMultimodalPrompt } from '../../../common';
+import type { ModelRuntime } from '../../models';
+export declare function AiExtractElementInfo<T>(options: {
+    dataQuery: string | Record<string, string>;
+    multimodalPrompt?: TMultimodalPrompt;
+    context: UIContext;
+    pageDescription?: string;
+    extractOption?: ServiceExtractOption;
+    modelRuntime: ModelRuntime;
+    abortSignal?: AbortSignal;
+}): Promise<{
+    parseResult: import("../../../types").AIDataExtractionResponse<T>;
+    rawResponse: string;
+    rawChoiceMessage: unknown;
+    usage: import("../../../types").AIUsageInfo | undefined;
+    reasoning_content: string | undefined;
+}>;

package/dist/types/ai-model/workflows/insight/index.d.ts

@@ -0,0 +1,2 @@
+export { AiExtractElementInfo } from './extraction';
+export { AiJudgeOrderSensitive } from './order-sensitive';

package/dist/types/ai-model/workflows/insight/insight-response-parser.d.ts

@@ -0,0 +1,4 @@
+import type { AIDataExtractionResponse } from '../../../types';
+import type { InsightDataOutputProtocol } from '../../model-adapter/insight-protocol';
+import type { JsonParser } from '../../shared/json';
+export declare function parseInsightResponse<T>(content: string, dataOutputProtocol: InsightDataOutputProtocol, jsonParser: JsonParser): AIDataExtractionResponse<T>;

package/dist/types/ai-model/workflows/insight/order-sensitive.d.ts

@@ -0,0 +1,6 @@
+import type { AIUsageInfo } from '../../../types';
+import type { ModelRuntime } from '../../models';
+export declare function AiJudgeOrderSensitive(description: string, modelRuntime: ModelRuntime): Promise<{
+    isOrderSensitive: boolean;
+    usage?: AIUsageInfo;
+}>;

package/dist/types/ai-model/workflows/planning/conversation-history.d.ts

@@ -0,0 +1,105 @@
+import type { SubGoal } from '../../../types';
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+export interface ConversationHistoryOptions {
+    initialMessages?: ChatCompletionMessageParam[];
+}
+export declare class ConversationHistory {
+    private readonly messages;
+    private subGoals;
+    private memories;
+    private historicalLogs;
+    pendingFeedbackMessage: string;
+    constructor(options?: ConversationHistoryOptions);
+    resetPendingFeedbackMessageIfExists(): void;
+    append(message: ChatCompletionMessageParam): void;
+    seed(messages: ChatCompletionMessageParam[]): void;
+    reset(): void;
+    /**
+     * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.
+     * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.
+     * @returns The snapshot of the conversation history.
+     */
+    snapshot(maxImages?: number): ChatCompletionMessageParam[];
+    get length(): number;
+    [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam>;
+    toJSON(): ChatCompletionMessageParam[];
+    /**
+     * Set all sub-goals, replacing any existing ones.
+     * Automatically marks the first pending goal as running.
+     */
+    setSubGoals(subGoals: SubGoal[]): void;
+    /**
+     * Merge sub-goals from update-plan-content.
+     * Preserves existing descriptions when incoming description is empty.
+     *
+     * This handles compact XML updates like:
+     * <sub-goal index="1" status="finished" />
+     */
+    mergeSubGoals(subGoals: SubGoal[]): void;
+    /**
+     * Update a single sub-goal by index.
+     * Clears logs if status or description actually changes.
+     * @returns true if the sub-goal was found and updated, false otherwise
+     */
+    updateSubGoal(index: number, updates: Partial<Omit<SubGoal, 'index'>>): boolean;
+    /**
+     * Mark the first pending sub-goal as running.
+     * Clears logs since status changes.
+     */
+    markFirstPendingAsRunning(): void;
+    /**
+     * Mark a sub-goal as finished.
+     * Automatically marks the next pending goal as running.
+     * @returns true if the sub-goal was found and updated, false otherwise
+     */
+    markSubGoalFinished(index: number): boolean;
+    /**
+     * Mark all sub-goals as finished.
+     * Clears logs for any goal whose status actually changes.
+     */
+    markAllSubGoalsFinished(): void;
+    /**
+     * Append a log entry to the currently running sub-goal.
+     * The log describes an action performed while working on the sub-goal.
+     */
+    appendSubGoalLog(log: string): void;
+    /**
+     * Convert sub-goals to text representation.
+     * Includes actions performed (logs) for the current sub-goal.
+     */
+    subGoalsToText(): string;
+    /**
+     * Append a log entry to the historical logs list.
+     * Used in non-deepThink mode to track executed steps across planning rounds.
+     */
+    appendHistoricalLog(log: string): void;
+    /**
+     * Convert historical logs to text representation.
+     * Provides context about previously executed steps to the model.
+     */
+    historicalLogsToText(): string;
+    /**
+     * Append a memory to the memories list
+     */
+    appendMemory(memory: string): void;
+    /**
+     * Get all memories
+     */
+    getMemories(): string[];
+    /**
+     * Convert memories to text representation
+     */
+    memoriesToText(): string;
+    /**
+     * Clear all memories
+     */
+    clearMemories(): void;
+    /**
+     * Compress the conversation history if it exceeds the threshold.
+     * Removes the oldest messages and replaces them with a single placeholder message.
+     * @param threshold - The number of messages that triggers compression.
+     * @param keepCount - The number of recent messages to keep after compression.
+     * @returns true if compression was performed, false otherwise.
+     */
+    compressHistory(threshold: number, keepCount: number): boolean;
+}

package/dist/types/ai-model/workflows/planning/custom-planning.d.ts

@@ -0,0 +1,7 @@
+import type { PlanningAIResponse } from '../../../types';
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+import type { CustomPlanningInput, CustomPlanningMessageConfig, ResolvedCustomPlanningDefinition } from '../../model-adapter/custom-planning-types';
+import { type PreparedUserPrompt } from '../../shared/multimodal-prompt';
+import type { PlanOptions } from './types';
+export declare function buildCustomPlanningMessages<TParsed>(input: CustomPlanningInput, config: CustomPlanningMessageConfig<TParsed>): ChatCompletionMessageParam[];
+export declare function runCustomPlanning<TParsed>(userInstruction: PreparedUserPrompt, options: PlanOptions, config: ResolvedCustomPlanningDefinition<TParsed>): Promise<PlanningAIResponse>;

package/dist/types/ai-model/workflows/planning/index.d.ts

@@ -0,0 +1,3 @@
+export { ConversationHistory, type ConversationHistoryOptions, } from './conversation-history';
+export { standardPlan } from './standard-planning';
+export { parseStandardPlanningResponse, parseXMLPlanningResponse, } from './standard-planning-parser';

package/dist/types/ai-model/workflows/planning/locate-normalization.d.ts

@@ -0,0 +1,12 @@
+import type { DeviceAction } from '../../../device';
+import type { PlanningAction } from '../../../types';
+import type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol';
+import type { LocateResultCodec, LocateResultContext } from '../../shared/model-locate-result/types';
+export declare function normalizePlanningActionLocateFields(actions: PlanningAction[], { actionSpace, includeLocateInPlanning, locateResultCodec, locateResultContext, acceptBbox2dAlias, parseRawLocateParameter, }: {
+    actionSpace: DeviceAction[];
+    includeLocateInPlanning: boolean;
+    locateResultCodec?: LocateResultCodec;
+    locateResultContext: LocateResultContext;
+    acceptBbox2dAlias?: boolean;
+    parseRawLocateParameter: (value: unknown) => ParsedPlanningLocateParameter;
+}): void;

package/dist/types/ai-model/workflows/planning/planning-action-log.d.ts

@@ -0,0 +1,2 @@
+import type { DeviceAction, PlanningAction } from '../../../types';
+export declare function buildPlanningActionLog(action: PlanningAction, actionSpace: DeviceAction<any>[]): string;

package/dist/types/ai-model/workflows/planning/standard-planning-parser.d.ts

@@ -0,0 +1,31 @@
+import type { DeviceAction, RawResponsePlanningAIResponse, SubGoal } from '../../../types';
+import type { PlanningActionOutputProtocol } from '../../model-adapter/planning-protocol';
+/**
+ * Parse sub-goals from XML content.
+ * Handles both formats:
+ * - <sub-goal index="1" status="pending">description</sub-goal>
+ * - <sub-goal index="1" status="finished" />
+ */
+export declare function parseSubGoalsFromXML(xmlContent: string): SubGoal[];
+/**
+ * Extract indexes of sub-goals marked as finished from
+ * <mark-sub-goal-done> content.
+ */
+export declare function parseMarkFinishedIndexes(xmlContent: string): number[];
+type XMLPlanningResponse = Omit<RawResponsePlanningAIResponse, 'action'>;
+type XMLPlanningResponseParseResult = {
+    parsed: XMLPlanningResponse;
+    rawActionOutput: string;
+};
+/** Parse common XML fields and retain the raw action-protocol XML. */
+export declare function parseXMLPlanningResponse(xmlString: string, actionOutputTagNames: PlanningActionOutputProtocol['actionOutputTagNames'], options: {
+    includeThought: boolean;
+}): XMLPlanningResponseParseResult;
+type ParseStandardPlanningResponseOptions = {
+    includeThought: boolean;
+    actionOutputProtocol: PlanningActionOutputProtocol;
+    actionSpace: DeviceAction<any>[];
+    logSource?: 'model' | 'action';
+};
+export declare function parseStandardPlanningResponse(xmlString: string, options: ParseStandardPlanningResponseOptions): RawResponsePlanningAIResponse;
+export {};

package/dist/types/ai-model/workflows/planning/standard-planning.d.ts

@@ -0,0 +1,4 @@
+import type { PlanningAIResponse } from '../../../types';
+import { type PreparedUserPrompt } from '../../shared/multimodal-prompt';
+import type { PlanOptions } from './types';
+export declare function standardPlan(userInstruction: PreparedUserPrompt, opts: PlanOptions): Promise<PlanningAIResponse>;

package/dist/types/ai-model/workflows/planning/types.d.ts

@@ -0,0 +1,16 @@
+import type { AiActEffort, DeviceAction, PlanningAIResponse, UIContext } from '../../../types';
+import type { ModelRuntime } from '../../models';
+import type { PreparedUserPrompt } from '../../shared/multimodal-prompt';
+import type { ConversationHistory } from './conversation-history';
+export interface PlanOptions {
+    context: UIContext;
+    actionSpace: DeviceAction<any>[];
+    actionContext?: string;
+    modelRuntime: ModelRuntime;
+    conversationHistory: ConversationHistory;
+    includeLocateInPlanning: boolean;
+    imagesIncludeCount?: number;
+    effort: AiActEffort;
+    abortSignal?: AbortSignal;
+}
+export type PlanFn = (userInstruction: PreparedUserPrompt, options: PlanOptions) => Promise<PlanningAIResponse>;