@aiscene/core 9.0.6 → 9.0.8

package/dist/es/ai-model/workflows/planning/custom-planning.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/planning/custom-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n  CustomPlanningInput,\n  CustomPlanningMessageConfig,\n  ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../../service-caller/index';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendHighPriorityKnowledge(\n  systemPrompt: string,\n  actionContext?: string,\n): string {\n  return (\n    systemPrompt +\n    (actionContext\n      ? `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\\n`\n      : '')\n  );\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n  input: CustomPlanningInput,\n  config: CustomPlanningMessageConfig<TParsed>,\n): ChatCompletionMessageParam[] {\n  const { options } = input;\n  const { conversationHistory, context, actionContext } = options;\n  const systemPrompt = appendHighPriorityKnowledge(\n    config.buildSystemPrompt(),\n    actionContext,\n  );\n  const userInstructionText = input.userInstruction.text;\n  const userInstruction = config.buildUserInstruction\n    ? config.buildUserInstruction(userInstructionText)\n    : userInstructionText;\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    input.userInstruction.referenceImages,\n  );\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    conversationHistory.append({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n        },\n      ],\n    });\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  }\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: { url: context.screenshot.base64 },\n      },\n    ],\n  });\n\n  if (config.systemPromptPlacement === 'system-message') {\n    return [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [{ type: 'text', text: userInstruction }],\n      },\n      ...referenceImageMessages,\n      ...conversationHistory.snapshot(config.historyImageLimit),\n    ];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: `${systemPrompt}${userInstruction}`,\n    },\n    ...referenceImageMessages,\n    ...conversationHistory.snapshot(config.historyImageLimit),\n  ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n  userInstruction: PreparedUserPrompt,\n  options: PlanOptions,\n  config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n  const { context } = options;\n  const preparedImage = await prepareModelImage({\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: options.modelRuntime.adapter.imagePreprocess,\n  });\n  const preparedOptions: PlanOptions = {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        preparedImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: preparedImage.preparedSize,\n    },\n  };\n  const input: CustomPlanningInput = {\n    userInstruction,\n    options: preparedOptions,\n    coordinateSystem: config.coordinateSystem,\n  };\n\n  const messages = buildCustomPlanningMessages(input, config.messages);\n  const { content, usage, rawChoiceMessage } = await callAIWithStringResponse(\n    messages,\n    preparedOptions.modelRuntime,\n    {\n      abortSignal: preparedOptions.abortSignal,\n      requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n    },\n  );\n\n  let parsed: TParsed;\n  let actions: PlanningAction[];\n  let shouldContinuePlanning: boolean;\n\n  try {\n    parsed = config.parseResponse(content, input);\n    actions = config.transformActions(parsed, input);\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: preparedOptions.actionSpace,\n      includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n      locateResultCodec: config.coordinateNormalizer,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n      acceptBbox2dAlias: preparedOptions.modelRuntime.adapter.acceptBbox2dAlias,\n      parseRawLocateParameter: (value) =>\n        value as ParsedPlanningLocateParameter,\n    });\n    shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n  } catch (parseError) {\n    const errorMessage = `Parse error: ${\n      parseError instanceof Error ? parseError.message : String(parseError)\n    }`;\n    throw new AIResponseParseError(\n      errorMessage,\n      JSON.stringify(content, undefined, 2),\n      usage,\n      rawChoiceMessage,\n    );\n  }\n\n  const assistantContent = config.messages.buildAssistantContent?.(\n    parsed,\n    content,\n    input,\n  );\n  // UI-TARS and Auto-GLM define their own assistant history format.\n  if (assistantContent) {\n    options.conversationHistory.append({\n      role: 'assistant',\n      content: assistantContent,\n    });\n  }\n\n  return {\n    actions,\n    log: config.buildResponseLog(parsed, content),\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(content, undefined, 2),\n    rawChoiceMessage,\n  };\n}\n"],"names":["appendHighPriorityKnowledge","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","conversationHistory","context","userInstructionText","userInstruction","referenceImageMessages","preparedReferenceImagesToChatMessages","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","messages","content","usage","rawChoiceMessage","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","value","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;;;AAqBA,SAASA,4BACPC,YAAoB,EACpBC,aAAsB;IAEtB,OACED,eACCC,CAAAA,gBACG,CAAC,yBAAyB,EAAEA,cAAc,4BAA4B,CAAC,GACvE,EAAC;AAET;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAE,GAAGF;IACpB,MAAM,EAAEG,mBAAmB,EAAEC,OAAO,EAAEN,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,4BACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMO,sBAAsBL,MAAM,eAAe,CAAC,IAAI;IACtD,MAAMM,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACI,uBAC5BA;IACJ,MAAME,yBAAyBC,sCAC7BR,MAAM,eAAe,CAAC,eAAe;IAGvC,IAAIG,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBAAE,KAAKC,QAAQ,UAAU,CAAC,MAAM;gBAAC;YAC9C;SACD;IACH;IAEA,IAAIH,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeQ,kBACpBH,eAAmC,EACnCJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMQ,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQF,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMU,kBAA+B;QACnC,GAAGV,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYS,eAAe,MAAM,CAC/BH,cAAc,WAAW,EACzBN,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUM,cAAc,YAAY;QACtC;IACF;IACA,MAAMV,QAA6B;QACjCM;QACA,SAASM;QACT,kBAAkBX,OAAO,gBAAgB;IAC3C;IAEA,MAAMa,WAAWf,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEc,OAAO,EAAEC,KAAK,EAAEC,gBAAgB,EAAE,GAAG,MAAMC,yBACjDJ,UACAF,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIO;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASlB,OAAO,aAAa,CAACc,SAASf;QACvCoB,UAAUnB,OAAO,gBAAgB,CAACkB,QAAQnB;QAC1CsB,oCAAoCF,SAAS;YAC3C,aAAaR,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,mBAAmBX,OAAO,oBAAoB;YAC9C,qBAAqB;gBACnB,cAAcS,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;YACA,mBAAmBE,gBAAgB,YAAY,CAAC,OAAO,CAAC,iBAAiB;YACzE,yBAAyB,CAACW,QACxBA;QACJ;QACAF,yBAAyBpB,OAAO,sBAAsB,CAACkB,QAAQC;IACjE,EAAE,OAAOI,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,qBACRH,cACAI,KAAK,SAAS,CAACd,SAASe,QAAW,IACnCd,OACAC;IAEJ;IAEA,MAAMc,mBAAmB9B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DkB,QACAJ,SACAf;IAGF,IAAI+B,kBACF7B,QAAQ,mBAAmB,CAAC,MAAM,CAAC;QACjC,MAAM;QACN,SAAS6B;IACX;IAGF,OAAO;QACLX;QACA,KAAKnB,OAAO,gBAAgB,CAACkB,QAAQJ;QACrCC;QACAK;QACA,aAAaQ,KAAK,SAAS,CAACd,SAASe,QAAW;QAChDb;IACF;AACF"}

package/dist/es/ai-model/workflows/planning/index.mjs

@@ -0,0 +1,4 @@
+import { ConversationHistory } from "./conversation-history.mjs";
+import { standardPlan } from "./standard-planning.mjs";
+import { parseStandardPlanningResponse, parseXMLPlanningResponse } from "./standard-planning-parser.mjs";
+export { ConversationHistory, parseStandardPlanningResponse, parseXMLPlanningResponse, standardPlan };

package/dist/es/ai-model/workflows/planning/locate-normalization.mjs

@@ -0,0 +1,41 @@
+import { findAllMidsceneLocatorField } from "../../../common.mjs";
+import { getDebug } from "@aiscene/shared/logger";
+import { assert } from "@aiscene/shared/utils";
+const debug = getDebug('planning');
+function normalizePlanningActionLocateFields(actions, { actionSpace, includeLocateInPlanning, locateResultCodec, locateResultContext, acceptBbox2dAlias = false, parseRawLocateParameter }) {
+    actions.forEach((action)=>{
+        const actionInActionSpace = actionSpace.find((actionInSpace)=>actionInSpace.name === action.type);
+        if (!actionInActionSpace) return void debug('skip locate normalization for action outside actionSpace', action);
+        debug('actionInActionSpace matched', actionInActionSpace);
+        const locateFields = findAllMidsceneLocatorField(actionInActionSpace.paramSchema);
+        debug('locateFields', locateFields);
+        locateFields.forEach((field)=>{
+            const rawLocateParameter = action.param?.[field];
+            if (!rawLocateParameter) return;
+            const locateParameter = parseRawLocateParameter(rawLocateParameter);
+            if (!includeLocateInPlanning) {
+                action.param[field] = {
+                    prompt: locateParameter.prompt
+                };
+                return;
+            }
+            assert(locateResultCodec, 'planning locate normalization requires a locate result codec');
+            const resultKey = locateResultCodec.promptSpec.resultKey;
+            const rawLocateValue = void 0 !== locateParameter[resultKey] ? locateParameter[resultKey] : acceptBbox2dAlias && 'bbox' === resultKey ? locateParameter.bbox_2d : void 0;
+            const rawCoordinateKeys = new Set([
+                resultKey,
+                ...acceptBbox2dAlias && 'bbox' === resultKey ? [
+                    'bbox_2d'
+                ] : []
+            ]);
+            const locateParamWithoutRawCoordinates = Object.fromEntries(Object.entries(locateParameter).filter(([key])=>!rawCoordinateKeys.has(key)));
+            action.param[field] = {
+                ...locateParamWithoutRawCoordinates,
+                locatedPixelBbox: locateResultCodec.toPixelBbox(rawLocateValue, locateResultContext)
+            };
+        });
+    });
+}
+export { normalizePlanningActionLocateFields };
+
+//# sourceMappingURL=locate-normalization.mjs.map

package/dist/es/ai-model/workflows/planning/locate-normalization.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/planning/locate-normalization.mjs","sources":["../../../../../src/ai-model/workflows/planning/locate-normalization.ts"],"sourcesContent":["import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { assert } from '@aiscene/shared/utils';\nimport type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result/types';\n\nconst debug = getDebug('planning');\n\nexport function normalizePlanningActionLocateFields(\n  actions: PlanningAction[],\n  {\n    actionSpace,\n    includeLocateInPlanning,\n    locateResultCodec,\n    locateResultContext,\n    acceptBbox2dAlias = false,\n    parseRawLocateParameter,\n  }: {\n    actionSpace: DeviceAction[];\n    includeLocateInPlanning: boolean;\n    locateResultCodec?: LocateResultCodec;\n    locateResultContext: LocateResultContext;\n    acceptBbox2dAlias?: boolean;\n    parseRawLocateParameter: (value: unknown) => ParsedPlanningLocateParameter;\n  },\n): void {\n  actions.forEach((action) => {\n    const actionInActionSpace = actionSpace.find(\n      (actionInSpace) => actionInSpace.name === action.type,\n    );\n    if (!actionInActionSpace) {\n      debug('skip locate normalization for action outside actionSpace', action);\n      return;\n    }\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = findAllMidsceneLocatorField(\n      actionInActionSpace.paramSchema,\n    );\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const rawLocateParameter = action.param?.[field];\n      if (!rawLocateParameter) {\n        return;\n      }\n\n      const locateParameter = parseRawLocateParameter(rawLocateParameter);\n\n      if (!includeLocateInPlanning) {\n        // In prompt-only planning mode, ignore any accidental coordinates from the model.\n        action.param[field] = { prompt: locateParameter.prompt };\n        return;\n      }\n\n      assert(\n        locateResultCodec,\n        'planning locate normalization requires a locate result codec',\n      );\n\n      const resultKey = locateResultCodec.promptSpec.resultKey;\n      const rawLocateValue =\n        locateParameter[resultKey] !== undefined\n          ? locateParameter[resultKey]\n          : acceptBbox2dAlias && resultKey === 'bbox'\n            ? locateParameter.bbox_2d\n            : undefined;\n\n      // The raw result field is replaced by locatedPixelBbox, so it should not\n      // remain in the normalized locate parameter.\n      const rawCoordinateKeys = new Set([\n        resultKey,\n        ...(acceptBbox2dAlias && resultKey === 'bbox' ? ['bbox_2d'] : []),\n      ]);\n\n      const locateParamWithoutRawCoordinates = Object.fromEntries(\n        Object.entries(locateParameter).filter(\n          ([key]) => !rawCoordinateKeys.has(key),\n        ),\n      );\n\n      action.param[field] = {\n        ...locateParamWithoutRawCoordinates,\n        locatedPixelBbox: locateResultCodec.toPixelBbox(\n          rawLocateValue,\n          locateResultContext,\n        ),\n      };\n    });\n  });\n}\n"],"names":["debug","getDebug","normalizePlanningActionLocateFields","actions","actionSpace","includeLocateInPlanning","locateResultCodec","locateResultContext","acceptBbox2dAlias","parseRawLocateParameter","action","actionInActionSpace","actionInSpace","locateFields","findAllMidsceneLocatorField","field","rawLocateParameter","locateParameter","assert","resultKey","rawLocateValue","undefined","rawCoordinateKeys","Set","locateParamWithoutRawCoordinates","Object","key"],"mappings":";;;AAWA,MAAMA,QAAQC,SAAS;AAEhB,SAASC,oCACdC,OAAyB,EACzB,EACEC,WAAW,EACXC,uBAAuB,EACvBC,iBAAiB,EACjBC,mBAAmB,EACnBC,oBAAoB,KAAK,EACzBC,uBAAuB,EAQxB;IAEDN,QAAQ,OAAO,CAAC,CAACO;QACf,MAAMC,sBAAsBP,YAAY,IAAI,CAC1C,CAACQ,gBAAkBA,cAAc,IAAI,KAAKF,OAAO,IAAI;QAEvD,IAAI,CAACC,qBAAqB,YACxBX,MAAM,4DAA4DU;QAIpEV,MAAM,+BAA+BW;QACrC,MAAME,eAAeC,4BACnBH,oBAAoB,WAAW;QAGjCX,MAAM,gBAAgBa;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,qBAAqBN,OAAO,KAAK,EAAE,CAACK,MAAM;YAChD,IAAI,CAACC,oBACH;YAGF,MAAMC,kBAAkBR,wBAAwBO;YAEhD,IAAI,CAACX,yBAAyB;gBAE5BK,OAAO,KAAK,CAACK,MAAM,GAAG;oBAAE,QAAQE,gBAAgB,MAAM;gBAAC;gBACvD;YACF;YAEAC,OACEZ,mBACA;YAGF,MAAMa,YAAYb,kBAAkB,UAAU,CAAC,SAAS;YACxD,MAAMc,iBACJH,AAA+BI,WAA/BJ,eAAe,CAACE,UAAU,GACtBF,eAAe,CAACE,UAAU,GAC1BX,qBAAqBW,AAAc,WAAdA,YACnBF,gBAAgB,OAAO,GACvBI;YAIR,MAAMC,oBAAoB,IAAIC,IAAI;gBAChCJ;mBACIX,qBAAqBW,AAAc,WAAdA,YAAuB;oBAAC;iBAAU,GAAG,EAAE;aACjE;YAED,MAAMK,mCAAmCC,OAAO,WAAW,CACzDA,OAAO,OAAO,CAACR,iBAAiB,MAAM,CACpC,CAAC,CAACS,IAAI,GAAK,CAACJ,kBAAkB,GAAG,CAACI;YAItChB,OAAO,KAAK,CAACK,MAAM,GAAG;gBACpB,GAAGS,gCAAgC;gBACnC,kBAAkBlB,kBAAkB,WAAW,CAC7Cc,gBACAb;YAEJ;QACF;IACF;AACF"}

package/dist/es/ai-model/workflows/planning/planning-action-log.mjs

@@ -0,0 +1,22 @@
+import { dumpActionParam } from "../../../common.mjs";
+const formatActionParamValue = (value)=>{
+    if ('string' == typeof value) return value;
+    if (null == value) return String(value);
+    return JSON.stringify(value) ?? String(value);
+};
+function buildPlanningActionLog(action, actionSpace) {
+    const actionDefinition = actionSpace.find((item)=>item.name === action.type);
+    if (!actionDefinition?.paramSchema) return JSON.stringify(action);
+    const param = action.param;
+    if (null == param) return action.type;
+    if ('object' != typeof param || Array.isArray(param)) return `${action.type} - ${formatActionParamValue(param)}`;
+    const paramForLog = dumpActionParam(param, actionDefinition.paramSchema);
+    const paramDescription = Object.entries(paramForLog).map(([key, value])=>`${key}: ${formatActionParamValue(value)}`).join(', ');
+    return [
+        action.type,
+        paramDescription
+    ].filter(Boolean).join(' - ');
+}
+export { buildPlanningActionLog };
+
+//# sourceMappingURL=planning-action-log.mjs.map

package/dist/es/ai-model/workflows/planning/planning-action-log.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/planning/planning-action-log.mjs","sources":["../../../../../src/ai-model/workflows/planning/planning-action-log.ts"],"sourcesContent":["import { dumpActionParam } from '@/common';\nimport type { DeviceAction, PlanningAction } from '@/types';\n\nconst formatActionParamValue = (value: unknown): string => {\n  if (typeof value === 'string') {\n    return value;\n  }\n  if (value === null || value === undefined) {\n    return String(value);\n  }\n  return JSON.stringify(value) ?? String(value);\n};\n\nexport function buildPlanningActionLog(\n  action: PlanningAction,\n  actionSpace: DeviceAction<any>[],\n): string {\n  const actionDefinition = actionSpace.find(\n    (item) => item.name === action.type,\n  );\n\n  if (!actionDefinition?.paramSchema) {\n    return JSON.stringify(action);\n  }\n\n  const param = action.param;\n  if (param === null || param === undefined) {\n    return action.type;\n  }\n  if (typeof param !== 'object' || Array.isArray(param)) {\n    return `${action.type} - ${formatActionParamValue(param)}`;\n  }\n\n  const paramForLog = dumpActionParam(param, actionDefinition.paramSchema);\n  const paramDescription = Object.entries(paramForLog)\n    .map(([key, value]) => `${key}: ${formatActionParamValue(value)}`)\n    .join(', ');\n\n  return [action.type, paramDescription].filter(Boolean).join(' - ');\n}\n"],"names":["formatActionParamValue","value","String","JSON","buildPlanningActionLog","action","actionSpace","actionDefinition","item","param","Array","paramForLog","dumpActionParam","paramDescription","Object","key","Boolean"],"mappings":";AAGA,MAAMA,yBAAyB,CAACC;IAC9B,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAET,IAAIA,QAAAA,OACF,OAAOC,OAAOD;IAEhB,OAAOE,KAAK,SAAS,CAACF,UAAUC,OAAOD;AACzC;AAEO,SAASG,uBACdC,MAAsB,EACtBC,WAAgC;IAEhC,MAAMC,mBAAmBD,YAAY,IAAI,CACvC,CAACE,OAASA,KAAK,IAAI,KAAKH,OAAO,IAAI;IAGrC,IAAI,CAACE,kBAAkB,aACrB,OAAOJ,KAAK,SAAS,CAACE;IAGxB,MAAMI,QAAQJ,OAAO,KAAK;IAC1B,IAAII,QAAAA,OACF,OAAOJ,OAAO,IAAI;IAEpB,IAAI,AAAiB,YAAjB,OAAOI,SAAsBC,MAAM,OAAO,CAACD,QAC7C,OAAO,GAAGJ,OAAO,IAAI,CAAC,GAAG,EAAEL,uBAAuBS,QAAQ;IAG5D,MAAME,cAAcC,gBAAgBH,OAAOF,iBAAiB,WAAW;IACvE,MAAMM,mBAAmBC,OAAO,OAAO,CAACH,aACrC,GAAG,CAAC,CAAC,CAACI,KAAKd,MAAM,GAAK,GAAGc,IAAI,EAAE,EAAEf,uBAAuBC,QAAQ,EAChE,IAAI,CAAC;IAER,OAAO;QAACI,OAAO,IAAI;QAAEQ;KAAiB,CAAC,MAAM,CAACG,SAAS,IAAI,CAAC;AAC9D"}

package/dist/es/ai-model/workflows/planning/standard-planning-parser.mjs

@@ -0,0 +1,99 @@
+import { extractRawXMLFragment, extractXMLTag } from "../../shared/xml.mjs";
+import { buildPlanningActionLog } from "./planning-action-log.mjs";
+function parseSubGoalsFromXML(xmlContent) {
+    const subGoals = [];
+    const regex = /<sub-goal\s+index="(\d+)"\s+status="(pending|finished)"(?:\s*\/>|>([\s\S]*?)<\/sub-goal>)/gi;
+    let match = regex.exec(xmlContent);
+    while(null !== match){
+        const index = Number.parseInt(match[1], 10);
+        const status = match[2];
+        const description = match[3]?.trim() || '';
+        subGoals.push({
+            index,
+            status,
+            description
+        });
+        match = regex.exec(xmlContent);
+    }
+    return subGoals;
+}
+function parseMarkFinishedIndexes(xmlContent) {
+    const indexes = [];
+    const regex = /<sub-goal\s+index="(\d+)"\s+status="finished"\s*\/>/gi;
+    let match = regex.exec(xmlContent);
+    while(null !== match){
+        indexes.push(Number.parseInt(match[1], 10));
+        match = regex.exec(xmlContent);
+    }
+    return indexes;
+}
+function parseXMLPlanningResponse(xmlString, actionOutputTagNames, options) {
+    const thought = options.includeThought ? extractXMLTag(xmlString, 'planning') : void 0;
+    const memory = extractXMLTag(xmlString, 'memory');
+    const log = extractXMLTag(xmlString, 'log') || '';
+    const error = extractXMLTag(xmlString, 'error');
+    const completeGoalRegex = /<complete\s+success="(true|false)">([\s\S]*?)<\/complete>/i;
+    const completeGoalMatch = xmlString.match(completeGoalRegex);
+    let finalizeMessage;
+    let finalizeSuccess;
+    if (completeGoalMatch) {
+        finalizeSuccess = 'true' === completeGoalMatch[1];
+        finalizeMessage = completeGoalMatch[2]?.trim() || void 0;
+    }
+    const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');
+    const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');
+    const updateSubGoals = updatePlanContent ? parseSubGoalsFromXML(updatePlanContent) : void 0;
+    const markFinishedIndexes = markSubGoalDone ? parseMarkFinishedIndexes(markSubGoalDone) : void 0;
+    return {
+        parsed: {
+            ...thought ? {
+                thought
+            } : {},
+            ...memory ? {
+                memory
+            } : {},
+            log,
+            ...error ? {
+                error
+            } : {},
+            ...void 0 !== finalizeMessage ? {
+                finalizeMessage
+            } : {},
+            ...void 0 !== finalizeSuccess ? {
+                finalizeSuccess
+            } : {},
+            ...updateSubGoals?.length ? {
+                updateSubGoals
+            } : {},
+            ...markFinishedIndexes?.length ? {
+                markFinishedIndexes
+            } : {}
+        },
+        rawActionOutput: extractRawXMLFragment(xmlString, actionOutputTagNames)
+    };
+}
+function buildNonActionPlanningLog(response) {
+    if (response.error) return `Error - ${response.error}`;
+    if (void 0 !== response.finalizeSuccess) return [
+        `Complete - success: ${response.finalizeSuccess}`,
+        response.finalizeMessage ? `message: ${response.finalizeMessage}` : void 0
+    ].filter(Boolean).join(', ');
+    return 'No action';
+}
+function parseStandardPlanningResponse(xmlString, options) {
+    const { parsed, rawActionOutput } = parseXMLPlanningResponse(xmlString, options.actionOutputProtocol.actionOutputTagNames, {
+        includeThought: options.includeThought
+    });
+    const response = {
+        ...parsed,
+        action: options.actionOutputProtocol.parseActionOutput(rawActionOutput, options.actionSpace)
+    };
+    if ('action' !== options.logSource) return response;
+    return {
+        ...response,
+        log: response.action ? buildPlanningActionLog(response.action, options.actionSpace) : buildNonActionPlanningLog(response)
+    };
+}
+export { parseMarkFinishedIndexes, parseStandardPlanningResponse, parseSubGoalsFromXML, parseXMLPlanningResponse };
+
+//# sourceMappingURL=standard-planning-parser.mjs.map

package/dist/es/ai-model/workflows/planning/standard-planning-parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/planning/standard-planning-parser.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning-parser.ts"],"sourcesContent":["import type {\n  DeviceAction,\n  RawResponsePlanningAIResponse,\n  SubGoal,\n  SubGoalStatus,\n} from '@/types';\nimport type { PlanningActionOutputProtocol } from '../../model-adapter/planning-protocol';\nimport { extractRawXMLFragment, extractXMLTag } from '../../shared/xml';\nimport { buildPlanningActionLog } from './planning-action-log';\n\n/**\n * Parse sub-goals from XML content.\n * Handles both formats:\n * - <sub-goal index=\"1\" status=\"pending\">description</sub-goal>\n * - <sub-goal index=\"1\" status=\"finished\" />\n */\nexport function parseSubGoalsFromXML(xmlContent: string): SubGoal[] {\n  const subGoals: SubGoal[] = [];\n  const regex =\n    /<sub-goal\\s+index=\"(\\d+)\"\\s+status=\"(pending|finished)\"(?:\\s*\\/>|>([\\s\\S]*?)<\\/sub-goal>)/gi;\n\n  let match = regex.exec(xmlContent);\n  while (match !== null) {\n    const index = Number.parseInt(match[1], 10);\n    const status = match[2] as SubGoalStatus;\n    const description = match[3]?.trim() || '';\n\n    subGoals.push({ index, status, description });\n    match = regex.exec(xmlContent);\n  }\n\n  return subGoals;\n}\n\n/**\n * Extract indexes of sub-goals marked as finished from\n * <mark-sub-goal-done> content.\n */\nexport function parseMarkFinishedIndexes(xmlContent: string): number[] {\n  const indexes: number[] = [];\n  const regex = /<sub-goal\\s+index=\"(\\d+)\"\\s+status=\"finished\"\\s*\\/>/gi;\n\n  let match = regex.exec(xmlContent);\n  while (match !== null) {\n    indexes.push(Number.parseInt(match[1], 10));\n    match = regex.exec(xmlContent);\n  }\n\n  return indexes;\n}\n\ntype XMLPlanningResponse = Omit<RawResponsePlanningAIResponse, 'action'>;\n\ntype XMLPlanningResponseParseResult = {\n  parsed: XMLPlanningResponse;\n  rawActionOutput: string;\n};\n\n/** Parse common XML fields and retain the raw action-protocol XML. */\nexport function parseXMLPlanningResponse(\n  xmlString: string,\n  actionOutputTagNames: PlanningActionOutputProtocol['actionOutputTagNames'],\n  options: { includeThought: boolean },\n): XMLPlanningResponseParseResult {\n  // Use <planning> instead of <thought> to avoid colliding with Gemini thought\n  // summaries, which may also be emitted as <thought> in OpenAI-compatible\n  // response content.\n  const thought = options.includeThought\n    ? extractXMLTag(xmlString, 'planning')\n    : undefined;\n  const memory = extractXMLTag(xmlString, 'memory');\n  const log = extractXMLTag(xmlString, 'log') || '';\n  const error = extractXMLTag(xmlString, 'error');\n\n  const completeGoalRegex =\n    /<complete\\s+success=\"(true|false)\">([\\s\\S]*?)<\\/complete>/i;\n  const completeGoalMatch = xmlString.match(completeGoalRegex);\n  let finalizeMessage: string | undefined;\n  let finalizeSuccess: boolean | undefined;\n\n  if (completeGoalMatch) {\n    finalizeSuccess = completeGoalMatch[1] === 'true';\n    finalizeMessage = completeGoalMatch[2]?.trim() || undefined;\n  }\n\n  const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n  const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n  const updateSubGoals = updatePlanContent\n    ? parseSubGoalsFromXML(updatePlanContent)\n    : undefined;\n  const markFinishedIndexes = markSubGoalDone\n    ? parseMarkFinishedIndexes(markSubGoalDone)\n    : undefined;\n\n  return {\n    parsed: {\n      ...(thought ? { thought } : {}),\n      ...(memory ? { memory } : {}),\n      log,\n      ...(error ? { error } : {}),\n      ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n      ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n      ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n      ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n    },\n    rawActionOutput: extractRawXMLFragment(xmlString, actionOutputTagNames),\n  };\n}\n\ntype ParseStandardPlanningResponseOptions = {\n  includeThought: boolean;\n  actionOutputProtocol: PlanningActionOutputProtocol;\n  actionSpace: DeviceAction<any>[];\n  logSource?: 'model' | 'action';\n};\n\nfunction buildNonActionPlanningLog(\n  response: RawResponsePlanningAIResponse,\n): string {\n  if (response.error) {\n    return `Error - ${response.error}`;\n  }\n\n  if (response.finalizeSuccess !== undefined) {\n    return [\n      `Complete - success: ${response.finalizeSuccess}`,\n      response.finalizeMessage\n        ? `message: ${response.finalizeMessage}`\n        : undefined,\n    ]\n      .filter(Boolean)\n      .join(', ');\n  }\n\n  return 'No action';\n}\n\nexport function parseStandardPlanningResponse(\n  xmlString: string,\n  options: ParseStandardPlanningResponseOptions,\n): RawResponsePlanningAIResponse {\n  const { parsed, rawActionOutput } = parseXMLPlanningResponse(\n    xmlString,\n    options.actionOutputProtocol.actionOutputTagNames,\n    { includeThought: options.includeThought },\n  );\n  const response: RawResponsePlanningAIResponse = {\n    ...parsed,\n    action: options.actionOutputProtocol.parseActionOutput(\n      rawActionOutput,\n      options.actionSpace,\n    ),\n  };\n\n  if (options.logSource !== 'action') {\n    return response;\n  }\n\n  return {\n    ...response,\n    log: response.action\n      ? buildPlanningActionLog(response.action, options.actionSpace)\n      : buildNonActionPlanningLog(response),\n  };\n}\n"],"names":["parseSubGoalsFromXML","xmlContent","subGoals","regex","match","index","Number","status","description","parseMarkFinishedIndexes","indexes","parseXMLPlanningResponse","xmlString","actionOutputTagNames","options","thought","extractXMLTag","undefined","memory","log","error","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","updatePlanContent","markSubGoalDone","updateSubGoals","markFinishedIndexes","extractRawXMLFragment","buildNonActionPlanningLog","response","Boolean","parseStandardPlanningResponse","parsed","rawActionOutput","buildPlanningActionLog"],"mappings":";;AAgBO,SAASA,qBAAqBC,UAAkB;IACrD,MAAMC,WAAsB,EAAE;IAC9B,MAAMC,QACJ;IAEF,IAAIC,QAAQD,MAAM,IAAI,CAACF;IACvB,MAAOG,AAAU,SAAVA,MAAgB;QACrB,MAAMC,QAAQC,OAAO,QAAQ,CAACF,KAAK,CAAC,EAAE,EAAE;QACxC,MAAMG,SAASH,KAAK,CAAC,EAAE;QACvB,MAAMI,cAAcJ,KAAK,CAAC,EAAE,EAAE,UAAU;QAExCF,SAAS,IAAI,CAAC;YAAEG;YAAOE;YAAQC;QAAY;QAC3CJ,QAAQD,MAAM,IAAI,CAACF;IACrB;IAEA,OAAOC;AACT;AAMO,SAASO,yBAAyBR,UAAkB;IACzD,MAAMS,UAAoB,EAAE;IAC5B,MAAMP,QAAQ;IAEd,IAAIC,QAAQD,MAAM,IAAI,CAACF;IACvB,MAAOG,AAAU,SAAVA,MAAgB;QACrBM,QAAQ,IAAI,CAACJ,OAAO,QAAQ,CAACF,KAAK,CAAC,EAAE,EAAE;QACvCA,QAAQD,MAAM,IAAI,CAACF;IACrB;IAEA,OAAOS;AACT;AAUO,SAASC,yBACdC,SAAiB,EACjBC,oBAA0E,EAC1EC,OAAoC;IAKpC,MAAMC,UAAUD,QAAQ,cAAc,GAClCE,cAAcJ,WAAW,cACzBK;IACJ,MAAMC,SAASF,cAAcJ,WAAW;IACxC,MAAMO,MAAMH,cAAcJ,WAAW,UAAU;IAC/C,MAAMQ,QAAQJ,cAAcJ,WAAW;IAEvC,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUL;IACpD;IAEA,MAAMQ,oBAAoBT,cAAcJ,WAAW;IACnD,MAAMc,kBAAkBV,cAAcJ,WAAW;IACjD,MAAMe,iBAAiBF,oBACnBzB,qBAAqByB,qBACrBR;IACJ,MAAMW,sBAAsBF,kBACxBjB,yBAAyBiB,mBACzBT;IAEJ,OAAO;QACL,QAAQ;YACN,GAAIF,UAAU;gBAAEA;YAAQ,IAAI,CAAC,CAAC;YAC9B,GAAIG,SAAS;gBAAEA;YAAO,IAAI,CAAC,CAAC;YAC5BC;YACA,GAAIC,QAAQ;gBAAEA;YAAM,IAAI,CAAC,CAAC;YAC1B,GAAIG,AAAoBN,WAApBM,kBAAgC;gBAAEA;YAAgB,IAAI,CAAC,CAAC;YAC5D,GAAIC,AAAoBP,WAApBO,kBAAgC;gBAAEA;YAAgB,IAAI,CAAC,CAAC;YAC5D,GAAIG,gBAAgB,SAAS;gBAAEA;YAAe,IAAI,CAAC,CAAC;YACpD,GAAIC,qBAAqB,SAAS;gBAAEA;YAAoB,IAAI,CAAC,CAAC;QAChE;QACA,iBAAiBC,sBAAsBjB,WAAWC;IACpD;AACF;AASA,SAASiB,0BACPC,QAAuC;IAEvC,IAAIA,SAAS,KAAK,EAChB,OAAO,CAAC,QAAQ,EAAEA,SAAS,KAAK,EAAE;IAGpC,IAAIA,AAA6Bd,WAA7Bc,SAAS,eAAe,EAC1B,OAAO;QACL,CAAC,oBAAoB,EAAEA,SAAS,eAAe,EAAE;QACjDA,SAAS,eAAe,GACpB,CAAC,SAAS,EAAEA,SAAS,eAAe,EAAE,GACtCd;KACL,CACE,MAAM,CAACe,SACP,IAAI,CAAC;IAGV,OAAO;AACT;AAEO,SAASC,8BACdrB,SAAiB,EACjBE,OAA6C;IAE7C,MAAM,EAAEoB,MAAM,EAAEC,eAAe,EAAE,GAAGxB,yBAClCC,WACAE,QAAQ,oBAAoB,CAAC,oBAAoB,EACjD;QAAE,gBAAgBA,QAAQ,cAAc;IAAC;IAE3C,MAAMiB,WAA0C;QAC9C,GAAGG,MAAM;QACT,QAAQpB,QAAQ,oBAAoB,CAAC,iBAAiB,CACpDqB,iBACArB,QAAQ,WAAW;IAEvB;IAEA,IAAIA,AAAsB,aAAtBA,QAAQ,SAAS,EACnB,OAAOiB;IAGT,OAAO;QACL,GAAGA,QAAQ;QACX,KAAKA,SAAS,MAAM,GAChBK,uBAAuBL,SAAS,MAAM,EAAEjB,QAAQ,WAAW,IAC3DgB,0BAA0BC;IAChC;AACF"}

package/dist/es/ai-model/workflows/planning/standard-planning.mjs

@@ -0,0 +1,219 @@
+import { getDebug } from "@aiscene/shared/logger";
+import { assert } from "@aiscene/shared/utils";
+import { buildYamlFlowFromPlans } from "../../../common.mjs";
+import { prepareModelImage } from "../../model-adapter/image-preprocess.mjs";
+import { buildStandardPlanningSystemPrompt } from "../../prompt/planning/index.mjs";
+import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
+import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
+import { planningModelFamilyRequiredForLocateMessage } from "../../shared/model-locate-result/errors.mjs";
+import { preparedReferenceImagesToChatMessages } from "../../shared/multimodal-prompt.mjs";
+import { normalizePlanningActionLocateFields } from "./locate-normalization.mjs";
+import { parseStandardPlanningResponse } from "./standard-planning-parser.mjs";
+const debug = getDebug('planning');
+const warnLog = getDebug('planning', {
+    console: true
+});
+const noPreviousActionsText = 'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';
+async function callAndParsePlanningResponse(options) {
+    const { messages, modelRuntime, abortSignal, includeLocateInPlanning, actionSpace, locateResultCodec, locateResultContext, includeThought, includeLog } = options;
+    assert('standard' === modelRuntime.adapter.planning.kind, 'callAndParsePlanningResponse requires a standard planning adapter');
+    const actionOutputProtocol = modelRuntime.adapter.planning.protocol.actionOutputProtocol;
+    return callAiAndParseWithRetry({
+        callAi: (retryAttempt, previousParseError)=>callAI(withSemanticRetryFeedback(messages, previousParseError), modelRuntime, {
+                abortSignal,
+                requiresOriginalImageDetail: includeLocateInPlanning,
+                semanticRetryAttempt: retryAttempt
+            }),
+        parseResponse: (response)=>{
+            const planFromAI = parseStandardPlanningResponse(response.content, {
+                includeThought,
+                actionOutputProtocol,
+                actionSpace,
+                logSource: includeLog ? 'model' : 'action'
+            });
+            if (planFromAI.action && void 0 !== planFromAI.finalizeSuccess) {
+                warnLog('Planning response included both an action and <complete>; ignoring <complete> output.');
+                planFromAI.finalizeMessage = void 0;
+                planFromAI.finalizeSuccess = void 0;
+            }
+            const actions = planFromAI.action ? [
+                planFromAI.action
+            ] : [];
+            normalizePlanningActionLocateFields(actions, {
+                actionSpace,
+                includeLocateInPlanning,
+                locateResultCodec,
+                locateResultContext,
+                acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,
+                parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter
+            });
+            const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);
+            return {
+                response,
+                planFromAI,
+                actions,
+                yamlFlow
+            };
+        },
+        toParseError: (parseError, response)=>{
+            const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
+            return new AIResponseParseError(`XML parse error: ${errorMessage}`, response.content, response.usage, response.rawChoiceMessage, response.reasoning_content);
+        },
+        parseRetryTimes: modelRuntime.config.retryCount,
+        parseRetryInterval: modelRuntime.config.retryInterval,
+        abortSignal,
+        onParseRetry: (parseError)=>{
+            debug('retrying plan after response parsing failed: %s', parseError instanceof Error ? parseError.message : String(parseError));
+        }
+    });
+}
+async function standardPlan(userInstruction, opts) {
+    const { context, conversationHistory } = opts;
+    const modelRuntime = opts.modelRuntime;
+    const { adapter } = modelRuntime;
+    const { shotSize } = context;
+    const screenshotBase64 = context.screenshot.base64;
+    assert('standard' === adapter.planning.kind, 'standardPlan requires a standard planning adapter');
+    const planningProtocol = adapter.planning.protocol;
+    if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) throw new Error(planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot));
+    const locateResultCodec = modelRuntime.config.modelFamily ? adapter.planning.locateResultCodec : void 0;
+    const includeSubGoals = 'deepThink' === opts.effort;
+    const includeThought = 'fast' !== opts.effort;
+    const includeLog = 'fast' !== opts.effort;
+    if (opts.includeLocateInPlanning && !locateResultCodec) throw new Error(planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot));
+    const systemPrompt = await buildStandardPlanningSystemPrompt({
+        actionSpace: opts.actionSpace,
+        includeThought,
+        includeLog,
+        includeSubGoals,
+        planningProtocol,
+        ...opts.includeLocateInPlanning && locateResultCodec ? {
+            includeLocateInPlanning: true,
+            locatePromptSpec: locateResultCodec.promptSpec
+        } : {
+            includeLocateInPlanning: false
+        }
+    });
+    const preparedImage = await prepareModelImage({
+        imageBase64: screenshotBase64,
+        width: shotSize.width,
+        height: shotSize.height,
+        policy: adapter.imagePreprocess
+    });
+    const imagePayload = preparedImage.imageBase64;
+    const actionContext = opts.actionContext ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\n` : '';
+    const referenceImageMessages = preparedReferenceImagesToChatMessages(userInstruction.referenceImages);
+    const instruction = [
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`
+                }
+            ]
+        },
+        ...referenceImageMessages
+    ];
+    let latestFeedbackMessage;
+    const executionProgressText = includeSubGoals ? conversationHistory.subGoalsToText() : conversationHistory.historicalLogsToText();
+    const executionProgressSection = executionProgressText ? `\n\n${executionProgressText}` : conversationHistory.pendingFeedbackMessage ? '' : `\n\n${noPreviousActionsText}`;
+    const memoriesText = conversationHistory.memoriesToText();
+    const memoriesSection = memoriesText ? `\n\n${memoriesText}` : '';
+    if (conversationHistory.pendingFeedbackMessage) {
+        latestFeedbackMessage = {
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`
+                },
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: imagePayload,
+                        detail: 'high'
+                    }
+                }
+            ]
+        };
+        conversationHistory.resetPendingFeedbackMessageIfExists();
+    } else latestFeedbackMessage = {
+        role: 'user',
+        content: [
+            {
+                type: 'text',
+                text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`
+            },
+            {
+                type: 'image_url',
+                image_url: {
+                    url: imagePayload,
+                    detail: 'high'
+                }
+            }
+        ]
+    };
+    conversationHistory.append(latestFeedbackMessage);
+    conversationHistory.compressHistory(50, 20);
+    const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        ...instruction,
+        ...historyLog
+    ];
+    const { response: { content: rawResponse, usage, reasoning_content, rawChoiceMessage }, planFromAI, actions, yamlFlow } = await callAndParsePlanningResponse({
+        messages: msgs,
+        modelRuntime,
+        abortSignal: opts.abortSignal,
+        includeLocateInPlanning: opts.includeLocateInPlanning,
+        actionSpace: opts.actionSpace,
+        locateResultCodec,
+        locateResultContext: {
+            preparedSize: preparedImage.preparedSize,
+            contentSize: preparedImage.contentSize
+        },
+        includeThought,
+        includeLog
+    });
+    let shouldContinuePlanning = true;
+    if (void 0 !== planFromAI.finalizeSuccess) {
+        debug('task completed via <complete> tag, stop planning');
+        shouldContinuePlanning = false;
+        if (includeSubGoals) conversationHistory.markAllSubGoalsFinished();
+    }
+    const returnValue = {
+        ...planFromAI,
+        actions,
+        rawResponse,
+        rawChoiceMessage,
+        usage,
+        reasoning_content,
+        yamlFlow,
+        shouldContinuePlanning
+    };
+    assert(planFromAI, "can't get plans from AI");
+    if (includeSubGoals) {
+        if (planFromAI.updateSubGoals?.length) conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);
+        if (planFromAI.markFinishedIndexes?.length) for (const index of planFromAI.markFinishedIndexes)conversationHistory.markSubGoalFinished(index);
+        if (planFromAI.log) conversationHistory.appendSubGoalLog(planFromAI.log);
+    } else if (planFromAI.log) conversationHistory.appendHistoricalLog(planFromAI.log);
+    if (planFromAI.memory) conversationHistory.appendMemory(planFromAI.memory);
+    if (modelRuntime.adapter.chatCompletion.replayRawAssistantMessage && rawChoiceMessage) conversationHistory.append(rawChoiceMessage);
+    else conversationHistory.append({
+        role: 'assistant',
+        content: [
+            {
+                type: 'text',
+                text: rawResponse
+            }
+        ]
+    });
+    return returnValue;
+}
+export { standardPlan };
+
+//# sourceMappingURL=standard-planning.mjs.map

package/dist/es/ai-model/workflows/planning/standard-planning.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { assert } from '@aiscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n  yamlFlow: ReturnType<typeof buildYamlFlowFromPlans>;\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      normalizePlanningActionLocateFields(actions, {\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n        acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n      });\n      // dumpActionParam keeps only the locator prompt, so runtime-only\n      // locatedPixelBbox fields added during normalization are not serialized.\n      const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);\n      return { response, planFromAI, actions, yamlFlow };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext\n    ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n    : '';\n\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n    yamlFlow,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  // Some model providers require opaque assistant fields to be replayed\n  // verbatim in later turns. Keep this opt-in per model adapter so that an\n  // unverified provider does not receive non-standard response fields.\n  if (\n    modelRuntime.adapter.chatCompletion.replayRawAssistantMessage &&\n    rawChoiceMessage\n  ) {\n    conversationHistory.append(rawChoiceMessage as ChatCompletionMessageParam);\n  } else {\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","actions","normalizePlanningActionLocateFields","yamlFlow","buildYamlFlowFromPlans","parseError","errorMessage","Error","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToChatMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","shouldContinuePlanning","returnValue","index"],"mappings":";;;;;;;;;;;AA6BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAO5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IAAIS,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAMC,UAAUH,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DI,oCAAoCD,SAAS;gBAC3ChB;gBACAD;gBACAE;gBACAC;gBACA,mBAAmBL,aAAa,OAAO,CAAC,iBAAiB;gBACzD,yBAAyBS,qBAAqB,uBAAuB;YACvE;YAGA,MAAMY,WAAWC,uBAAuBH,SAAShB;YACjD,OAAO;gBAAEY;gBAAUC;gBAAYG;gBAASE;YAAS;QACnD;QACA,cAAc,CAACE,YAAYR;YACzB,MAAMS,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACsB;YACb9B,MACE,mDACA8B,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;QAE9D;IACF;AACF;AAEO,eAAeK,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM9B,eAAe8B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAGjC;IACpB,MAAM,EAAEkC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDvB,OACEyB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC9B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIyB,MACRY,4CAA4CrC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrDiC,QAAQ,QAAQ,CAAC,iBAAiB,GAClCf;IAGJ,MAAMoB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMxB,iBAAiBwB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMvB,aAAauB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAAC1B,mBACnC,MAAM,IAAIqB,MACRY,4CAA4CrC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMuC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BxB;QACAC;QACA+B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAI1B,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMqC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMe,yBAAyBC,sCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAEpC,uBAAuB;IAGpC,MAAMuD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKP;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKP;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFX,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACD1C,UAAU,EACVG,OAAO,EACPE,QAAQ,EACT,GAAG,MAAMxB,6BAA6B;QACrC,UAAUyD;QACVtD;QACA,aAAa8B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7B1B;QACA,qBAAqB;YACnB,cAAcqC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAnC;QACAC;IACF;IAEA,IAAIoD,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIrB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM4B,cAAkC;QACtC,GAAG5C,UAAU;QACbG;QACAoC;QACAG;QACAF;QACAC;QACApC;QACAsC;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIsB,iBAAiB;QACnB,IAAItB,WAAW,cAAc,EAAE,QAC7BgB,oBAAoB,aAAa,CAAChB,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDgB,oBAAoB,mBAAmB,CAAC6B;QAI5C,IAAI7C,WAAW,GAAG,EAChBgB,oBAAoB,gBAAgB,CAAChB,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBgB,oBAAoB,mBAAmB,CAAChB,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBgB,oBAAoB,YAAY,CAAChB,WAAW,MAAM;IAMpD,IACEhB,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB,IAC7D0D,kBAEA1B,oBAAoB,MAAM,CAAC0B;SAE3B1B,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAGF,OAAOK;AACT"}