@aiscene/core 9.0.6 → 9.0.8

package/dist/es/ai-model/workflows/grounding/section-locate.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/section-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["import type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { assert } from '@aiscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mergePixelBboxesToRect } from './locate-result-rect';\nimport { buildSearchAreaConfig, expandSearchArea } from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelBbox(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelBbox(reference, locateResultContext),\n          );\n          const mergedRect = mergePixelBboxesToRect([\n            target,\n            ...(references ?? []),\n          ]);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergePixelBboxesToRect","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;;;;;;;AAiBA,MAAMA,eAAeC,SAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,OAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,kCAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,mBAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,wBAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,WAAW,CACpCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,WAAW,CAACqB,WAAWH;oBAErC,MAAMI,aAAaC,uBAAuB;wBACxCJ;2BACIC,cAAc,EAAE;qBACrB;oBACD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,yBAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,qBACTD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,iBAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,sBAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/utils.mjs

@@ -0,0 +1,50 @@
+import { userPromptToMultimodalPrompt } from "../../../common.mjs";
+import { prepareModelImage } from "../../model-adapter/image-preprocess.mjs";
+import { multimodalPromptToChatMessages } from "../../shared/multimodal-prompt.mjs";
+async function prepareLocateModelInput({ locateImage, targetDescription, systemPrompt, userPrompt, imagePreprocess, userMessageContentOrder }) {
+    const preparedImage = await prepareModelImage({
+        imageBase64: locateImage.imageBase64,
+        width: locateImage.width,
+        height: locateImage.height,
+        policy: imagePreprocess
+    });
+    const referenceImageMessages = await multimodalPromptToChatMessages(userPromptToMultimodalPrompt(targetDescription));
+    const imageContent = {
+        type: 'image_url',
+        image_url: {
+            url: preparedImage.imageBase64,
+            detail: 'high'
+        }
+    };
+    const promptContent = {
+        type: 'text',
+        text: userPrompt
+    };
+    return {
+        preparedImage,
+        messages: [
+            {
+                role: 'system',
+                content: systemPrompt
+            },
+            {
+                role: 'user',
+                content: 'prompt-first' === userMessageContentOrder ? [
+                    promptContent,
+                    imageContent
+                ] : [
+                    imageContent,
+                    promptContent
+                ]
+            },
+            ...referenceImageMessages
+        ]
+    };
+}
+function formatLocateModelContext(modelRuntime) {
+    const { modelFamily, modelName } = modelRuntime.config;
+    return `modelName=${modelName ?? 'unset'} modelFamily=${modelFamily ?? 'unset'}`;
+}
+export { formatLocateModelContext, prepareLocateModelInput };
+
+//# sourceMappingURL=utils.mjs.map

package/dist/es/ai-model/workflows/grounding/utils.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/grounding/utils.mjs","sources":["../../../../../src/ai-model/workflows/grounding/utils.ts"],"sourcesContent":["import { type TUserPrompt, userPromptToMultimodalPrompt } from '@/common';\nimport type {\n  ChatCompletionContentPart,\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport {\n  type ImagePreprocessPolicy,\n  type PreparedModelImage,\n  prepareModelImage,\n} from '../../model-adapter/image-preprocess';\nimport type { LocateUserMessageContentOrder } from '../../model-adapter/types';\nimport type { ModelRuntime } from '../../models';\nimport { multimodalPromptToChatMessages } from '../../shared/multimodal-prompt';\n\nexport type GroundingAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nexport async function prepareLocateModelInput({\n  locateImage,\n  targetDescription,\n  systemPrompt,\n  userPrompt,\n  imagePreprocess,\n  userMessageContentOrder,\n}: {\n  locateImage: {\n    imageBase64: string;\n    width: number;\n    height: number;\n  };\n  targetDescription: TUserPrompt;\n  systemPrompt: string;\n  userPrompt: string;\n  imagePreprocess: ImagePreprocessPolicy;\n  userMessageContentOrder: LocateUserMessageContentOrder;\n}): Promise<{\n  messages: GroundingAIArgs;\n  preparedImage: PreparedModelImage;\n}> {\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateImage.imageBase64,\n    width: locateImage.width,\n    height: locateImage.height,\n    policy: imagePreprocess,\n  });\n  const referenceImageMessages = await multimodalPromptToChatMessages(\n    userPromptToMultimodalPrompt(targetDescription),\n  );\n  const imageContent: ChatCompletionContentPart = {\n    type: 'image_url',\n    image_url: {\n      url: preparedImage.imageBase64,\n      detail: 'high',\n    },\n  };\n  const promptContent: ChatCompletionContentPart = {\n    type: 'text',\n    text: userPrompt,\n  };\n\n  return {\n    preparedImage,\n    messages: [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content:\n          userMessageContentOrder === 'prompt-first'\n            ? [promptContent, imageContent]\n            : [imageContent, promptContent],\n      },\n      ...referenceImageMessages,\n    ],\n  };\n}\n\nexport function formatLocateModelContext(modelRuntime: ModelRuntime): string {\n  const { modelFamily, modelName } = modelRuntime.config;\n  return `modelName=${modelName ?? 'unset'} modelFamily=${modelFamily ?? 'unset'}`;\n}\n"],"names":["prepareLocateModelInput","locateImage","targetDescription","systemPrompt","userPrompt","imagePreprocess","userMessageContentOrder","preparedImage","prepareModelImage","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","imageContent","promptContent","formatLocateModelContext","modelRuntime","modelFamily","modelName"],"mappings":";;;AAoBO,eAAeA,wBAAwB,EAC5CC,WAAW,EACXC,iBAAiB,EACjBC,YAAY,EACZC,UAAU,EACVC,eAAe,EACfC,uBAAuB,EAYxB;IAIC,MAAMC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,YAAY,WAAW;QACpC,OAAOA,YAAY,KAAK;QACxB,QAAQA,YAAY,MAAM;QAC1B,QAAQI;IACV;IACA,MAAMI,yBAAyB,MAAMC,+BACnCC,6BAA6BT;IAE/B,MAAMU,eAA0C;QAC9C,MAAM;QACN,WAAW;YACT,KAAKL,cAAc,WAAW;YAC9B,QAAQ;QACV;IACF;IACA,MAAMM,gBAA2C;QAC/C,MAAM;QACN,MAAMT;IACR;IAEA,OAAO;QACLG;QACA,UAAU;YACR;gBAAE,MAAM;gBAAU,SAASJ;YAAa;YACxC;gBACE,MAAM;gBACN,SACEG,AAA4B,mBAA5BA,0BACI;oBAACO;oBAAeD;iBAAa,GAC7B;oBAACA;oBAAcC;iBAAc;YACrC;eACGJ;SACJ;IACH;AACF;AAEO,SAASK,yBAAyBC,YAA0B;IACjE,MAAM,EAAEC,WAAW,EAAEC,SAAS,EAAE,GAAGF,aAAa,MAAM;IACtD,OAAO,CAAC,UAAU,EAAEE,aAAa,QAAQ,aAAa,EAAED,eAAe,SAAS;AAClF"}

package/dist/es/ai-model/workflows/insight/extraction.mjs

@@ -0,0 +1,101 @@
+import { getDebug } from "@aiscene/shared/logger";
+import { buildInsightSystemPrompt, extractDataQueryPrompt } from "../../prompt/insight/index.mjs";
+import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
+import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
+import { multimodalPromptToChatMessages } from "../../shared/multimodal-prompt.mjs";
+import { parseInsightResponse } from "./insight-response-parser.mjs";
+const debugInsight = getDebug('ai:insight');
+async function AiExtractElementInfo(options) {
+    const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } = options;
+    const insightProtocol = modelRuntime.adapter.insight.protocol;
+    const systemPrompt = buildInsightSystemPrompt({
+        screenshotIncluded: extractOption?.screenshotIncluded !== false,
+        referenceImagesIncluded: !!multimodalPrompt?.images?.length,
+        insightProtocol
+    });
+    const screenshotBase64 = context.screenshot.base64;
+    const extractDataPromptText = extractDataQueryPrompt(options.pageDescription || '', dataQuery, extractOption?.context);
+    const userContent = [];
+    if (extractOption?.screenshotIncluded !== false) {
+        const screenshotSequence = context.screenshotSequence;
+        if (screenshotSequence && screenshotSequence.length > 1) {
+            userContent.push({
+                type: 'text',
+                text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`
+            });
+            screenshotSequence.forEach((frame, index)=>{
+                userContent.push({
+                    type: 'text',
+                    text: `Frame ${index + 1}/${screenshotSequence.length}`
+                });
+                userContent.push({
+                    type: 'image_url',
+                    image_url: {
+                        url: frame.base64,
+                        detail: 'high'
+                    }
+                });
+            });
+        } else {
+            userContent.push({
+                type: 'text',
+                text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.'
+            });
+            userContent.push({
+                type: 'image_url',
+                image_url: {
+                    url: screenshotBase64,
+                    detail: 'high'
+                }
+            });
+        }
+    }
+    userContent.push({
+        type: 'text',
+        text: extractDataPromptText
+    });
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userContent
+        }
+    ];
+    if (multimodalPrompt) {
+        const addOns = await multimodalPromptToChatMessages(multimodalPrompt);
+        msgs.push(...addOns);
+    }
+    return callAiAndParseWithRetry({
+        callAi: (retryAttempt, previousParseError)=>callAI(withSemanticRetryFeedback(msgs, previousParseError), modelRuntime, {
+                abortSignal: options.abortSignal,
+                semanticRetryAttempt: retryAttempt
+            }),
+        parseResponse: (response)=>{
+            const { content: rawResponse, usage, reasoning_content, rawChoiceMessage } = response;
+            const parseResult = parseInsightResponse(rawResponse, insightProtocol.dataOutput, modelRuntime.adapter.jsonParser);
+            return {
+                parseResult,
+                rawResponse,
+                rawChoiceMessage,
+                usage,
+                reasoning_content
+            };
+        },
+        toParseError: (parseError, response)=>{
+            const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
+            return new AIResponseParseError(`XML parse error: ${errorMessage}`, response.content, response.usage, response.rawChoiceMessage);
+        },
+        parseRetryTimes: modelRuntime.config.retryCount,
+        parseRetryInterval: modelRuntime.config.retryInterval,
+        abortSignal: options.abortSignal,
+        onParseRetry: (error)=>{
+            debugInsight('retrying insight after XML parsing failed: %s', error instanceof Error ? error.message : String(error));
+        }
+    });
+}
+export { AiExtractElementInfo };
+
+//# sourceMappingURL=extraction.mjs.map

package/dist/es/ai-model/workflows/insight/extraction.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/insight/extraction.mjs","sources":["../../../../../src/ai-model/workflows/insight/extraction.ts"],"sourcesContent":["import type { ServiceExtractOption, UIContext } from '@/types';\nimport { getDebug } from '@aiscene/shared/logger';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport {\n  buildInsightSystemPrompt,\n  extractDataQueryPrompt,\n} from '../../prompt/insight';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { multimodalPromptToChatMessages } from '../../shared/multimodal-prompt';\nimport { parseInsightResponse } from './insight-response-parser';\n\ntype InsightAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const insightProtocol = modelRuntime.adapter.insight.protocol;\n  const systemPrompt = buildInsightSystemPrompt({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n    insightProtocol,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    extractOption?.context as string | undefined,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image_url',\n          image_url: {\n            url: frame.base64,\n            detail: 'high',\n          },\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n          detail: 'high',\n        },\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(msgs, previousParseError),\n        modelRuntime,\n        {\n          abortSignal: options.abortSignal,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawChoiceMessage,\n      } = response;\n      const parseResult = parseInsightResponse<T>(\n        rawResponse,\n        insightProtocol.dataOutput,\n        modelRuntime.adapter.jsonParser,\n      );\n      return {\n        parseResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInsight(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n"],"names":["debugInsight","getDebug","AiExtractElementInfo","options","dataQuery","context","extractOption","multimodalPrompt","modelRuntime","insightProtocol","systemPrompt","buildInsightSystemPrompt","screenshotBase64","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","msgs","addOns","multimodalPromptToChatMessages","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","rawResponse","usage","reasoning_content","rawChoiceMessage","parseResult","parseInsightResponse","parseError","errorMessage","Error","String","AIResponseParseError","error"],"mappings":";;;;;;AAyBA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,qBAAwBC,OAQ7C;IACC,MAAM,EAAEC,SAAS,EAAEC,OAAO,EAAEC,aAAa,EAAEC,gBAAgB,EAAEC,YAAY,EAAE,GACzEL;IACF,MAAMM,kBAAkBD,aAAa,OAAO,CAAC,OAAO,CAAC,QAAQ;IAC7D,MAAME,eAAeC,yBAAyB;QAC5C,oBAAoBL,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;QACrDE;IACF;IACA,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAMQ,wBAAwBC,uBAC5BX,QAAQ,eAAe,IAAI,IAC3BC,WACAE,eAAe;IAGjB,MAAMS,cAAyD,EAAE;IAEjE,IAAIT,eAAe,uBAAuB,OAAO;QAC/C,MAAMU,qBAAqBX,QAAQ,kBAAkB;QACrD,IAAIW,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,WAAW;wBACT,KAAKE,MAAM,MAAM;wBACjB,QAAQ;oBACV;gBACF;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,WAAW;oBACT,KAAKH;oBACL,QAAQ;gBACV;YACF;QACF;IACF;IAEAG,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAAST;QAAa;QACxC;YACE,MAAM;YACN,SAASK;QACX;KACD;IAED,IAAIR,kBAAkB;QACpB,MAAMa,SAAS,MAAMC,+BAA+Bd;QACpDY,KAAK,IAAI,IAAIC;IACf;IAEA,OAAOE,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BP,MAAMK,qBAChChB,cACA;gBACE,aAAaL,QAAQ,WAAW;gBAChC,sBAAsBoB;YACxB;QAEJ,eAAe,CAACI;YACd,MAAM,EACJ,SAASC,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,GAAGJ;YACJ,MAAMK,cAAcC,qBAClBL,aACAnB,gBAAgB,UAAU,EAC1BD,aAAa,OAAO,CAAC,UAAU;YAEjC,OAAO;gBACLwB;gBACAJ;gBACAG;gBACAF;gBACAC;YACF;QACF;QACA,cAAc,CAACI,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB;QAE7B;QACA,iBAAiBnB,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAaL,QAAQ,WAAW;QAChC,cAAc,CAACoC;YACbvC,aACE,iDACAuC,iBAAiBH,QAAQG,MAAM,OAAO,GAAGF,OAAOE;QAEpD;IACF;AACF"}

package/dist/es/ai-model/workflows/insight/index.mjs

@@ -0,0 +1,3 @@
+import { AiExtractElementInfo } from "./extraction.mjs";
+import { AiJudgeOrderSensitive } from "./order-sensitive.mjs";
+export { AiExtractElementInfo, AiJudgeOrderSensitive };

package/dist/es/ai-model/workflows/insight/insight-response-parser.mjs

@@ -0,0 +1,26 @@
+import { extractRawXMLFragment, extractXMLTag } from "../../shared/xml.mjs";
+function parseInsightResponse(content, dataOutputProtocol, jsonParser) {
+    const observation = extractXMLTag(content, 'observation');
+    const errorsContent = extractXMLTag(content, 'errors');
+    const rawDataOutput = extractRawXMLFragment(content, dataOutputProtocol.tagNames);
+    const data = dataOutputProtocol.parse(rawDataOutput);
+    let errors;
+    if (errorsContent) try {
+        const parsedErrors = jsonParser(errorsContent, {
+            source: 'generic-object'
+        });
+        if (Array.isArray(parsedErrors)) errors = parsedErrors;
+    } catch  {}
+    return {
+        ...observation ? {
+            thought: observation
+        } : {},
+        data,
+        ...errors?.length ? {
+            errors
+        } : {}
+    };
+}
+export { parseInsightResponse };
+
+//# sourceMappingURL=insight-response-parser.mjs.map

package/dist/es/ai-model/workflows/insight/insight-response-parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/insight/insight-response-parser.mjs","sources":["../../../../../src/ai-model/workflows/insight/insight-response-parser.ts"],"sourcesContent":["import type { AIDataExtractionResponse } from '@/types';\nimport type { InsightDataOutputProtocol } from '../../model-adapter/insight-protocol';\nimport type { JsonParser } from '../../shared/json';\nimport { extractRawXMLFragment, extractXMLTag } from '../../shared/xml';\n\nexport function parseInsightResponse<T>(\n  content: string,\n  dataOutputProtocol: InsightDataOutputProtocol,\n  jsonParser: JsonParser,\n): AIDataExtractionResponse<T> {\n  const observation = extractXMLTag(content, 'observation');\n  const errorsContent = extractXMLTag(content, 'errors');\n  const rawDataOutput = extractRawXMLFragment(\n    content,\n    dataOutputProtocol.tagNames,\n  );\n  const data = dataOutputProtocol.parse<T>(rawDataOutput);\n\n  let errors: string[] | undefined;\n  if (errorsContent) {\n    try {\n      const parsedErrors = jsonParser(errorsContent, {\n        source: 'generic-object',\n      });\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch {\n      // Error details are optional and must not discard valid extracted data.\n    }\n  }\n\n  return {\n    ...(observation ? { thought: observation } : {}),\n    data,\n    ...(errors?.length ? { errors } : {}),\n  };\n}\n"],"names":["parseInsightResponse","content","dataOutputProtocol","jsonParser","observation","extractXMLTag","errorsContent","rawDataOutput","extractRawXMLFragment","data","errors","parsedErrors","Array"],"mappings":";AAKO,SAASA,qBACdC,OAAe,EACfC,kBAA6C,EAC7CC,UAAsB;IAEtB,MAAMC,cAAcC,cAAcJ,SAAS;IAC3C,MAAMK,gBAAgBD,cAAcJ,SAAS;IAC7C,MAAMM,gBAAgBC,sBACpBP,SACAC,mBAAmB,QAAQ;IAE7B,MAAMO,OAAOP,mBAAmB,KAAK,CAAIK;IAEzC,IAAIG;IACJ,IAAIJ,eACF,IAAI;QACF,MAAMK,eAAeR,WAAWG,eAAe;YAC7C,QAAQ;QACV;QACA,IAAIM,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAM,CAER;IAGF,OAAO;QACL,GAAIP,cAAc;YAAE,SAASA;QAAY,IAAI,CAAC,CAAC;QAC/CK;QACA,GAAIC,QAAQ,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;IACtC;AACF"}

package/dist/es/ai-model/workflows/insight/order-sensitive.mjs

@@ -0,0 +1,29 @@
+import { getDebug } from "@aiscene/shared/logger";
+import { orderSensitiveJudgePrompt, systemPromptToJudgeOrderSensitive } from "../../prompt/order-sensitive-judge.mjs";
+import { callAIWithObjectResponse } from "../../service-caller/index.mjs";
+const debugInsight = getDebug('ai:insight');
+async function AiJudgeOrderSensitive(description, modelRuntime) {
+    const systemPrompt = systemPromptToJudgeOrderSensitive();
+    const userPrompt = orderSensitiveJudgePrompt(description);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userPrompt
+        }
+    ];
+    debugInsight("AiJudgeOrderSensitive: description=%s", description);
+    const result = await callAIWithObjectResponse(msgs, modelRuntime, {
+        jsonParserSource: 'generic-object'
+    });
+    return {
+        isOrderSensitive: result.content.isOrderSensitive ?? false,
+        usage: result.usage
+    };
+}
+export { AiJudgeOrderSensitive };
+
+//# sourceMappingURL=order-sensitive.mjs.map

package/dist/es/ai-model/workflows/insight/order-sensitive.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/insight/order-sensitive.mjs","sources":["../../../../../src/ai-model/workflows/insight/order-sensitive.ts"],"sourcesContent":["import type { AIUsageInfo } from '@/types';\nimport { getDebug } from '@aiscene/shared/logger';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { ModelRuntime } from '../../models';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from '../../prompt/order-sensitive-judge';\nimport { callAIWithObjectResponse } from '../../service-caller/index';\n\ntype InsightAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInsight = getDebug('ai:insight');\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InsightAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInsight('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInsight","getDebug","AiJudgeOrderSensitive","description","modelRuntime","systemPrompt","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt","msgs","result","callAIWithObjectResponse"],"mappings":";;;AAkBA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,sBACpBC,WAAmB,EACnBC,YAA0B;IAK1B,MAAMC,eAAeC;IACrB,MAAMC,aAAaC,0BAA0BL;IAE7C,MAAMM,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAASE;QACX;KACD;IAEDP,aAAa,yCAAyCG;IAEtD,MAAMO,SAAS,MAAMC,yBACnBF,MACAL,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkBM,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/workflows/planning/conversation-history.mjs

@@ -0,0 +1,185 @@
+import { buildSubGoalsText } from "../../prompt/planning/sub-goals-text.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+var _computedKey;
+_computedKey = Symbol.iterator;
+let _computedKey1 = _computedKey;
+class ConversationHistory {
+    resetPendingFeedbackMessageIfExists() {
+        if (this.pendingFeedbackMessage) this.pendingFeedbackMessage = '';
+    }
+    append(message) {
+        this.messages.push(message);
+    }
+    seed(messages) {
+        this.reset();
+        messages.forEach((message)=>{
+            this.append(message);
+        });
+    }
+    reset() {
+        this.messages.length = 0;
+        this.memories.length = 0;
+        this.subGoals.length = 0;
+        this.historicalLogs.length = 0;
+        this.pendingFeedbackMessage = '';
+    }
+    snapshot(maxImages) {
+        if (void 0 === maxImages) return [
+            ...this.messages
+        ];
+        const clonedMessages = structuredClone(this.messages);
+        let imageCount = 0;
+        for(let i = clonedMessages.length - 1; i >= 0; i--){
+            const message = clonedMessages[i];
+            const content = message.content;
+            if (Array.isArray(content)) for(let j = 0; j < content.length; j++){
+                const item = content[j];
+                if ('image_url' === item.type) {
+                    imageCount++;
+                    if (imageCount > maxImages) content[j] = {
+                        type: 'text',
+                        text: '(image ignored due to size optimization)'
+                    };
+                }
+            }
+        }
+        return clonedMessages;
+    }
+    get length() {
+        return this.messages.length;
+    }
+    [_computedKey1]() {
+        return this.messages[Symbol.iterator]();
+    }
+    toJSON() {
+        return this.snapshot();
+    }
+    setSubGoals(subGoals) {
+        this.subGoals = subGoals.map((goal)=>({
+                ...goal
+            }));
+        this.markFirstPendingAsRunning();
+    }
+    mergeSubGoals(subGoals) {
+        if (0 === this.subGoals.length) return void this.setSubGoals(subGoals);
+        const existingByIndex = new Map(this.subGoals.map((goal)=>[
+                goal.index,
+                goal
+            ]));
+        const mergedSubGoals = subGoals.map((goal)=>{
+            const existingGoal = existingByIndex.get(goal.index);
+            const hasNonEmptyDescription = goal.description.trim().length > 0;
+            if (!existingGoal && !hasNonEmptyDescription) return null;
+            return {
+                ...goal,
+                description: hasNonEmptyDescription || !existingGoal ? goal.description : existingGoal.description
+            };
+        });
+        const validSubGoals = mergedSubGoals.filter((goal)=>null !== goal);
+        if (0 === validSubGoals.length) return;
+        this.setSubGoals(validSubGoals);
+    }
+    updateSubGoal(index, updates) {
+        const goal = this.subGoals.find((g)=>g.index === index);
+        if (!goal) return false;
+        let changed = false;
+        if (void 0 !== updates.status && updates.status !== goal.status) {
+            goal.status = updates.status;
+            changed = true;
+        }
+        if (void 0 !== updates.description && updates.description !== goal.description) {
+            goal.description = updates.description;
+            changed = true;
+        }
+        if (changed) goal.logs = [];
+        return true;
+    }
+    markFirstPendingAsRunning() {
+        const firstPending = this.subGoals.find((g)=>'pending' === g.status);
+        if (firstPending) {
+            firstPending.status = 'running';
+            firstPending.logs = [];
+        }
+    }
+    markSubGoalFinished(index) {
+        const result = this.updateSubGoal(index, {
+            status: 'finished'
+        });
+        if (result) this.markFirstPendingAsRunning();
+        return result;
+    }
+    markAllSubGoalsFinished() {
+        for (const goal of this.subGoals){
+            if ('finished' !== goal.status) goal.logs = [];
+            goal.status = 'finished';
+        }
+    }
+    appendSubGoalLog(log) {
+        if (!log) return;
+        const runningGoal = this.subGoals.find((g)=>'running' === g.status);
+        if (runningGoal) {
+            if (!runningGoal.logs) runningGoal.logs = [];
+            runningGoal.logs.push(log);
+        }
+    }
+    subGoalsToText() {
+        return buildSubGoalsText(this.subGoals);
+    }
+    appendHistoricalLog(log) {
+        if (log) this.historicalLogs.push(log);
+    }
+    historicalLogsToText() {
+        if (0 === this.historicalLogs.length) return '';
+        const logLines = this.historicalLogs.map((log)=>`- ${log}`).join('\n');
+        return `Here are the steps that have been executed:\n${logLines}`;
+    }
+    appendMemory(memory) {
+        if (memory) this.memories.push(memory);
+    }
+    getMemories() {
+        return [
+            ...this.memories
+        ];
+    }
+    memoriesToText() {
+        if (0 === this.memories.length) return '';
+        return `Memories from previous steps:\n---\n${this.memories.join('\n---\n')}\n`;
+    }
+    clearMemories() {
+        this.memories.length = 0;
+    }
+    compressHistory(threshold, keepCount) {
+        if (this.messages.length <= threshold) return false;
+        const omittedCount = this.messages.length - keepCount;
+        const omittedPlaceholder = {
+            role: 'user',
+            content: `(${omittedCount} previous conversation messages have been omitted)`
+        };
+        const recentMessages = this.messages.slice(-keepCount);
+        this.messages.length = 0;
+        this.messages.push(omittedPlaceholder);
+        for (const msg of recentMessages)this.messages.push(msg);
+        return true;
+    }
+    constructor(options){
+        _define_property(this, "messages", []);
+        _define_property(this, "subGoals", []);
+        _define_property(this, "memories", []);
+        _define_property(this, "historicalLogs", []);
+        _define_property(this, "pendingFeedbackMessage", void 0);
+        if (options?.initialMessages?.length) this.seed(options.initialMessages);
+        this.pendingFeedbackMessage = '';
+    }
+}
+export { ConversationHistory };
+
+//# sourceMappingURL=conversation-history.mjs.map

package/dist/es/ai-model/workflows/planning/conversation-history.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/planning/conversation-history.mjs","sources":["../../../../../src/ai-model/workflows/planning/conversation-history.ts"],"sourcesContent":["import type { SubGoal } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildSubGoalsText } from '../../prompt/planning/sub-goals-text';\n\nexport interface ConversationHistoryOptions {\n  initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n  private readonly messages: ChatCompletionMessageParam[] = [];\n  private subGoals: SubGoal[] = [];\n  private memories: string[] = [];\n  private historicalLogs: string[] = [];\n\n  public pendingFeedbackMessage: string;\n\n  constructor(options?: ConversationHistoryOptions) {\n    if (options?.initialMessages?.length) {\n      this.seed(options.initialMessages);\n    }\n    this.pendingFeedbackMessage = '';\n  }\n\n  resetPendingFeedbackMessageIfExists() {\n    if (this.pendingFeedbackMessage) {\n      this.pendingFeedbackMessage = '';\n    }\n  }\n\n  append(message: ChatCompletionMessageParam) {\n    this.messages.push(message);\n  }\n\n  seed(messages: ChatCompletionMessageParam[]) {\n    this.reset();\n    messages.forEach((message) => {\n      this.append(message);\n    });\n  }\n\n  reset() {\n    this.messages.length = 0;\n    this.memories.length = 0;\n    this.subGoals.length = 0;\n    this.historicalLogs.length = 0;\n    this.pendingFeedbackMessage = '';\n  }\n\n  /**\n   * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.\n   * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.\n   * @returns The snapshot of the conversation history.\n   */\n  snapshot(maxImages?: number): ChatCompletionMessageParam[] {\n    if (maxImages === undefined) {\n      return [...this.messages];\n    }\n\n    const clonedMessages = structuredClone(this.messages);\n    let imageCount = 0;\n\n    // Traverse from the end to the beginning\n    for (let i = clonedMessages.length - 1; i >= 0; i--) {\n      const message = clonedMessages[i];\n      const content = message.content;\n\n      // Only process if content is an array\n      if (Array.isArray(content)) {\n        for (let j = 0; j < content.length; j++) {\n          const item = content[j];\n\n          // Check if this is an image\n          if (item.type === 'image_url') {\n            imageCount++;\n\n            // If we've exceeded the limit, replace with text\n            if (imageCount > maxImages) {\n              content[j] = {\n                type: 'text',\n                text: '(image ignored due to size optimization)',\n              };\n            }\n          }\n        }\n      }\n    }\n\n    return clonedMessages;\n  }\n\n  get length(): number {\n    return this.messages.length;\n  }\n\n  [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n    return this.messages[Symbol.iterator]();\n  }\n\n  toJSON(): ChatCompletionMessageParam[] {\n    return this.snapshot();\n  }\n\n  // Planning sub-goal management methods\n\n  /**\n   * Set all sub-goals, replacing any existing ones.\n   * Automatically marks the first pending goal as running.\n   */\n  setSubGoals(subGoals: SubGoal[]): void {\n    this.subGoals = subGoals.map((goal) => ({ ...goal }));\n    this.markFirstPendingAsRunning();\n  }\n\n  /**\n   * Merge sub-goals from update-plan-content.\n   * Preserves existing descriptions when incoming description is empty.\n   *\n   * This handles compact XML updates like:\n   * <sub-goal index=\"1\" status=\"finished\" />\n   */\n  mergeSubGoals(subGoals: SubGoal[]): void {\n    if (this.subGoals.length === 0) {\n      this.setSubGoals(subGoals);\n      return;\n    }\n\n    const existingByIndex = new Map(\n      this.subGoals.map((goal) => [goal.index, goal] as const),\n    );\n\n    const mergedSubGoals = subGoals.map((goal) => {\n      const existingGoal = existingByIndex.get(goal.index);\n      const hasNonEmptyDescription = goal.description.trim().length > 0;\n\n      if (!existingGoal && !hasNonEmptyDescription) {\n        return null;\n      }\n\n      return {\n        ...goal,\n        description:\n          hasNonEmptyDescription || !existingGoal\n            ? goal.description\n            : existingGoal.description,\n      };\n    });\n\n    const validSubGoals = mergedSubGoals.filter((goal) => goal !== null);\n    if (validSubGoals.length === 0) {\n      return;\n    }\n\n    this.setSubGoals(validSubGoals);\n  }\n\n  /**\n   * Update a single sub-goal by index.\n   * Clears logs if status or description actually changes.\n   * @returns true if the sub-goal was found and updated, false otherwise\n   */\n  updateSubGoal(\n    index: number,\n    updates: Partial<Omit<SubGoal, 'index'>>,\n  ): boolean {\n    const goal = this.subGoals.find((g) => g.index === index);\n    if (!goal) {\n      return false;\n    }\n\n    let changed = false;\n\n    if (updates.status !== undefined && updates.status !== goal.status) {\n      goal.status = updates.status;\n      changed = true;\n    }\n    if (\n      updates.description !== undefined &&\n      updates.description !== goal.description\n    ) {\n      goal.description = updates.description;\n      changed = true;\n    }\n\n    if (changed) {\n      goal.logs = [];\n    }\n\n    return true;\n  }\n\n  /**\n   * Mark the first pending sub-goal as running.\n   * Clears logs since status changes.\n   */\n  markFirstPendingAsRunning(): void {\n    const firstPending = this.subGoals.find((g) => g.status === 'pending');\n    if (firstPending) {\n      firstPending.status = 'running';\n      firstPending.logs = [];\n    }\n  }\n\n  /**\n   * Mark a sub-goal as finished.\n   * Automatically marks the next pending goal as running.\n   * @returns true if the sub-goal was found and updated, false otherwise\n   */\n  markSubGoalFinished(index: number): boolean {\n    const result = this.updateSubGoal(index, { status: 'finished' });\n    if (result) {\n      this.markFirstPendingAsRunning();\n    }\n    return result;\n  }\n\n  /**\n   * Mark all sub-goals as finished.\n   * Clears logs for any goal whose status actually changes.\n   */\n  markAllSubGoalsFinished(): void {\n    for (const goal of this.subGoals) {\n      if (goal.status !== 'finished') {\n        goal.logs = [];\n      }\n      goal.status = 'finished';\n    }\n  }\n\n  /**\n   * Append a log entry to the currently running sub-goal.\n   * The log describes an action performed while working on the sub-goal.\n   */\n  appendSubGoalLog(log: string): void {\n    if (!log) {\n      return;\n    }\n    const runningGoal = this.subGoals.find((g) => g.status === 'running');\n    if (runningGoal) {\n      if (!runningGoal.logs) {\n        runningGoal.logs = [];\n      }\n      runningGoal.logs.push(log);\n    }\n  }\n\n  /**\n   * Convert sub-goals to text representation.\n   * Includes actions performed (logs) for the current sub-goal.\n   */\n  subGoalsToText(): string {\n    return buildSubGoalsText(this.subGoals);\n  }\n\n  // Historical log management methods (used in non-deepThink mode)\n\n  /**\n   * Append a log entry to the historical logs list.\n   * Used in non-deepThink mode to track executed steps across planning rounds.\n   */\n  appendHistoricalLog(log: string): void {\n    if (log) {\n      this.historicalLogs.push(log);\n    }\n  }\n\n  /**\n   * Convert historical logs to text representation.\n   * Provides context about previously executed steps to the model.\n   */\n  historicalLogsToText(): string {\n    if (this.historicalLogs.length === 0) {\n      return '';\n    }\n\n    const logLines = this.historicalLogs.map((log) => `- ${log}`).join('\\n');\n    return `Here are the steps that have been executed:\\n${logLines}`;\n  }\n\n  // Memory management methods\n\n  /**\n   * Append a memory to the memories list\n   */\n  appendMemory(memory: string): void {\n    if (memory) {\n      this.memories.push(memory);\n    }\n  }\n\n  /**\n   * Get all memories\n   */\n  getMemories(): string[] {\n    return [...this.memories];\n  }\n\n  /**\n   * Convert memories to text representation\n   */\n  memoriesToText(): string {\n    if (this.memories.length === 0) {\n      return '';\n    }\n\n    return `Memories from previous steps:\\n---\\n${this.memories.join('\\n---\\n')}\\n`;\n  }\n\n  /**\n   * Clear all memories\n   */\n  clearMemories(): void {\n    this.memories.length = 0;\n  }\n\n  /**\n   * Compress the conversation history if it exceeds the threshold.\n   * Removes the oldest messages and replaces them with a single placeholder message.\n   * @param threshold - The number of messages that triggers compression.\n   * @param keepCount - The number of recent messages to keep after compression.\n   * @returns true if compression was performed, false otherwise.\n   */\n  compressHistory(threshold: number, keepCount: number): boolean {\n    if (this.messages.length <= threshold) {\n      return false;\n    }\n\n    const omittedCount = this.messages.length - keepCount;\n    const omittedPlaceholder: ChatCompletionMessageParam = {\n      role: 'user',\n      content: `(${omittedCount} previous conversation messages have been omitted)`,\n    };\n\n    // Keep only the last `keepCount` messages\n    const recentMessages = this.messages.slice(-keepCount);\n\n    // Reset and rebuild with placeholder + recent messages\n    this.messages.length = 0;\n    this.messages.push(omittedPlaceholder);\n    for (const msg of recentMessages) {\n      this.messages.push(msg);\n    }\n\n    return true;\n  }\n}\n"],"names":["Symbol","ConversationHistory","message","messages","maxImages","undefined","clonedMessages","structuredClone","imageCount","i","content","Array","j","item","subGoals","goal","existingByIndex","Map","mergedSubGoals","existingGoal","hasNonEmptyDescription","validSubGoals","index","updates","g","changed","firstPending","result","log","runningGoal","buildSubGoalsText","logLines","memory","threshold","keepCount","omittedCount","omittedPlaceholder","recentMessages","msg","options"],"mappings":";;;;;;;;;;;;eA8FGA,OAAO,QAAQ;;AAtFX,MAAMC;IAeX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,OAAOC,OAAmC,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG;QAC7B,IAAI,CAAC,sBAAsB,GAAG;IAChC;IAOA,SAASE,SAAkB,EAAgC;QACzD,IAAIA,AAAcC,WAAdD,WACF,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;QAG3B,MAAME,iBAAiBC,gBAAgB,IAAI,CAAC,QAAQ;QACpD,IAAIC,aAAa;QAGjB,IAAK,IAAIC,IAAIH,eAAe,MAAM,GAAG,GAAGG,KAAK,GAAGA,IAAK;YACnD,MAAMP,UAAUI,cAAc,CAACG,EAAE;YACjC,MAAMC,UAAUR,QAAQ,OAAO;YAG/B,IAAIS,MAAM,OAAO,CAACD,UAChB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,QAAQ,MAAM,EAAEE,IAAK;gBACvC,MAAMC,OAAOH,OAAO,CAACE,EAAE;gBAGvB,IAAIC,AAAc,gBAAdA,KAAK,IAAI,EAAkB;oBAC7BL;oBAGA,IAAIA,aAAaJ,WACfM,OAAO,CAACE,EAAE,GAAG;wBACX,MAAM;wBACN,MAAM;oBACR;gBAEJ;YACF;QAEJ;QAEA,OAAON;IACT;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACN,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IAQA,YAAYc,QAAmB,EAAQ;QACrC,IAAI,CAAC,QAAQ,GAAGA,SAAS,GAAG,CAAC,CAACC,OAAU;gBAAE,GAAGA,IAAI;YAAC;QAClD,IAAI,CAAC,yBAAyB;IAChC;IASA,cAAcD,QAAmB,EAAQ;QACvC,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAQ,YAC9B,IAAI,CAAC,WAAW,CAACA;QAInB,MAAME,kBAAkB,IAAIC,IAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACF,OAAS;gBAACA,KAAK,KAAK;gBAAEA;aAAK;QAGhD,MAAMG,iBAAiBJ,SAAS,GAAG,CAAC,CAACC;YACnC,MAAMI,eAAeH,gBAAgB,GAAG,CAACD,KAAK,KAAK;YACnD,MAAMK,yBAAyBL,KAAK,WAAW,CAAC,IAAI,GAAG,MAAM,GAAG;YAEhE,IAAI,CAACI,gBAAgB,CAACC,wBACpB,OAAO;YAGT,OAAO;gBACL,GAAGL,IAAI;gBACP,aACEK,0BAA0B,CAACD,eACvBJ,KAAK,WAAW,GAChBI,aAAa,WAAW;YAChC;QACF;QAEA,MAAME,gBAAgBH,eAAe,MAAM,CAAC,CAACH,OAASA,AAAS,SAATA;QACtD,IAAIM,AAAyB,MAAzBA,cAAc,MAAM,EACtB;QAGF,IAAI,CAAC,WAAW,CAACA;IACnB;IAOA,cACEC,KAAa,EACbC,OAAwC,EAC/B;QACT,MAAMR,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACS,IAAMA,EAAE,KAAK,KAAKF;QACnD,IAAI,CAACP,MACH,OAAO;QAGT,IAAIU,UAAU;QAEd,IAAIF,AAAmBlB,WAAnBkB,QAAQ,MAAM,IAAkBA,QAAQ,MAAM,KAAKR,KAAK,MAAM,EAAE;YAClEA,KAAK,MAAM,GAAGQ,QAAQ,MAAM;YAC5BE,UAAU;QACZ;QACA,IACEF,AAAwBlB,WAAxBkB,QAAQ,WAAW,IACnBA,QAAQ,WAAW,KAAKR,KAAK,WAAW,EACxC;YACAA,KAAK,WAAW,GAAGQ,QAAQ,WAAW;YACtCE,UAAU;QACZ;QAEA,IAAIA,SACFV,KAAK,IAAI,GAAG,EAAE;QAGhB,OAAO;IACT;IAMA,4BAAkC;QAChC,MAAMW,eAAe,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACF,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACvD,IAAIE,cAAc;YAChBA,aAAa,MAAM,GAAG;YACtBA,aAAa,IAAI,GAAG,EAAE;QACxB;IACF;IAOA,oBAAoBJ,KAAa,EAAW;QAC1C,MAAMK,SAAS,IAAI,CAAC,aAAa,CAACL,OAAO;YAAE,QAAQ;QAAW;QAC9D,IAAIK,QACF,IAAI,CAAC,yBAAyB;QAEhC,OAAOA;IACT;IAMA,0BAAgC;QAC9B,KAAK,MAAMZ,QAAQ,IAAI,CAAC,QAAQ,CAAE;YAChC,IAAIA,AAAgB,eAAhBA,KAAK,MAAM,EACbA,KAAK,IAAI,GAAG,EAAE;YAEhBA,KAAK,MAAM,GAAG;QAChB;IACF;IAMA,iBAAiBa,GAAW,EAAQ;QAClC,IAAI,CAACA,KACH;QAEF,MAAMC,cAAc,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACL,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACtD,IAAIK,aAAa;YACf,IAAI,CAACA,YAAY,IAAI,EACnBA,YAAY,IAAI,GAAG,EAAE;YAEvBA,YAAY,IAAI,CAAC,IAAI,CAACD;QACxB;IACF;IAMA,iBAAyB;QACvB,OAAOE,kBAAkB,IAAI,CAAC,QAAQ;IACxC;IAQA,oBAAoBF,GAAW,EAAQ;QACrC,IAAIA,KACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAACA;IAE7B;IAMA,uBAA+B;QAC7B,IAAI,AAA+B,MAA/B,IAAI,CAAC,cAAc,CAAC,MAAM,EAC5B,OAAO;QAGT,MAAMG,WAAW,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAACH,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;QACnE,OAAO,CAAC,6CAA6C,EAAEG,UAAU;IACnE;IAOA,aAAaC,MAAc,EAAQ;QACjC,IAAIA,QACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IAEvB;IAKA,cAAwB;QACtB,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAKA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACjF;IAKA,gBAAsB;QACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IASA,gBAAgBC,SAAiB,EAAEC,SAAiB,EAAW;QAC7D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAID,WAC1B,OAAO;QAGT,MAAME,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAGD;QAC5C,MAAME,qBAAiD;YACrD,MAAM;YACN,SAAS,CAAC,CAAC,EAAED,aAAa,kDAAkD,CAAC;QAC/E;QAGA,MAAME,iBAAiB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAACH;QAG5C,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACE;QACnB,KAAK,MAAME,OAAOD,eAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACC;QAGrB,OAAO;IACT;IAvUA,YAAYC,OAAoC,CAAE;QAPlD,uBAAiB,YAAyC,EAAE;QAC5D,uBAAQ,YAAsB,EAAE;QAChC,uBAAQ,YAAqB,EAAE;QAC/B,uBAAQ,kBAA2B,EAAE;QAErC,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAmUF"}

package/dist/es/ai-model/workflows/planning/custom-planning.mjs

@@ -0,0 +1,129 @@
+import { ScreenshotItem } from "../../../screenshot-item.mjs";
+import { prepareModelImage } from "../../model-adapter/image-preprocess.mjs";
+import { AIResponseParseError, callAIWithStringResponse } from "../../service-caller/index.mjs";
+import { preparedReferenceImagesToChatMessages } from "../../shared/multimodal-prompt.mjs";
+import { normalizePlanningActionLocateFields } from "./locate-normalization.mjs";
+function appendHighPriorityKnowledge(systemPrompt, actionContext) {
+    return systemPrompt + (actionContext ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\n` : '');
+}
+function buildCustomPlanningMessages(input, config) {
+    const { options } = input;
+    const { conversationHistory, context, actionContext } = options;
+    const systemPrompt = appendHighPriorityKnowledge(config.buildSystemPrompt(), actionContext);
+    const userInstructionText = input.userInstruction.text;
+    const userInstruction = config.buildUserInstruction ? config.buildUserInstruction(userInstructionText) : userInstructionText;
+    const referenceImageMessages = preparedReferenceImagesToChatMessages(input.userInstruction.referenceImages);
+    if (conversationHistory.pendingFeedbackMessage) {
+        conversationHistory.append({
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`
+                }
+            ]
+        });
+        conversationHistory.resetPendingFeedbackMessageIfExists();
+    }
+    conversationHistory.append({
+        role: 'user',
+        content: [
+            {
+                type: 'image_url',
+                image_url: {
+                    url: context.screenshot.base64
+                }
+            }
+        ]
+    });
+    if ('system-message' === config.systemPromptPlacement) return [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: userInstruction
+                }
+            ]
+        },
+        ...referenceImageMessages,
+        ...conversationHistory.snapshot(config.historyImageLimit)
+    ];
+    return [
+        {
+            role: 'user',
+            content: `${systemPrompt}${userInstruction}`
+        },
+        ...referenceImageMessages,
+        ...conversationHistory.snapshot(config.historyImageLimit)
+    ];
+}
+async function runCustomPlanning(userInstruction, options, config) {
+    const { context } = options;
+    const preparedImage = await prepareModelImage({
+        imageBase64: context.screenshot.base64,
+        width: context.shotSize.width,
+        height: context.shotSize.height,
+        policy: options.modelRuntime.adapter.imagePreprocess
+    });
+    const preparedOptions = {
+        ...options,
+        context: {
+            ...context,
+            screenshot: ScreenshotItem.create(preparedImage.imageBase64, context.screenshot.capturedAt),
+            shotSize: preparedImage.preparedSize
+        }
+    };
+    const input = {
+        userInstruction,
+        options: preparedOptions,
+        coordinateSystem: config.coordinateSystem
+    };
+    const messages = buildCustomPlanningMessages(input, config.messages);
+    const { content, usage, rawChoiceMessage } = await callAIWithStringResponse(messages, preparedOptions.modelRuntime, {
+        abortSignal: preparedOptions.abortSignal,
+        requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning
+    });
+    let parsed;
+    let actions;
+    let shouldContinuePlanning;
+    try {
+        parsed = config.parseResponse(content, input);
+        actions = config.transformActions(parsed, input);
+        normalizePlanningActionLocateFields(actions, {
+            actionSpace: preparedOptions.actionSpace,
+            includeLocateInPlanning: preparedOptions.includeLocateInPlanning,
+            locateResultCodec: config.coordinateNormalizer,
+            locateResultContext: {
+                preparedSize: preparedImage.preparedSize,
+                contentSize: preparedImage.contentSize
+            },
+            acceptBbox2dAlias: preparedOptions.modelRuntime.adapter.acceptBbox2dAlias,
+            parseRawLocateParameter: (value)=>value
+        });
+        shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);
+    } catch (parseError) {
+        const errorMessage = `Parse error: ${parseError instanceof Error ? parseError.message : String(parseError)}`;
+        throw new AIResponseParseError(errorMessage, JSON.stringify(content, void 0, 2), usage, rawChoiceMessage);
+    }
+    const assistantContent = config.messages.buildAssistantContent?.(parsed, content, input);
+    if (assistantContent) options.conversationHistory.append({
+        role: 'assistant',
+        content: assistantContent
+    });
+    return {
+        actions,
+        log: config.buildResponseLog(parsed, content),
+        usage,
+        shouldContinuePlanning,
+        rawResponse: JSON.stringify(content, void 0, 2),
+        rawChoiceMessage
+    };
+}
+export { buildCustomPlanningMessages, runCustomPlanning };
+
+//# sourceMappingURL=custom-planning.mjs.map