@xagent-ai/cli 1.0.1

package/src/gui-subagent/action-parser/actionParser.ts

@@ -0,0 +1,312 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import {
+  ActionInputs,
+  PredictionParsed,
+  UITarsModelVersion,
+} from './types.js';
+import {
+  MAX_RATIO,
+  IMAGE_FACTOR,
+  MIN_PIXELS,
+  MAX_PIXELS_V1_5,
+} from './constants.js';
+import isNumber from 'lodash.isnumber';
+
+function roundByFactor(num: number, factor: number): number {
+  return Math.round(num / factor) * factor;
+}
+
+function floorByFactor(num: number, factor: number): number {
+  return Math.floor(num / factor) * factor;
+}
+
+function ceilByFactor(num: number, factor: number): number {
+  return Math.ceil(num / factor) * factor;
+}
+
+function smartResizeForV15(
+  height: number,
+  width: number,
+  maxRatio: number = MAX_RATIO,
+  factor: number = IMAGE_FACTOR,
+  minPixels: number = MIN_PIXELS,
+  maxPixels: number = MAX_PIXELS_V1_5,
+): [number, number] | null {
+  if (Math.max(height, width) / Math.min(height, width) > maxRatio) {
+    console.error(
+      `absolute aspect ratio must be smaller than ${maxRatio}, got ${
+        Math.max(height, width) / Math.min(height, width)
+      }`,
+    );
+    return null;
+  }
+
+  let wBar = Math.max(factor, roundByFactor(width, factor));
+  let hBar = Math.max(factor, roundByFactor(height, factor));
+
+  if (hBar * wBar > maxPixels) {
+    const beta = Math.sqrt((height * width) / maxPixels);
+    hBar = floorByFactor(height / beta, factor);
+    wBar = floorByFactor(width / beta, factor);
+  } else if (hBar * wBar < minPixels) {
+    const beta = Math.sqrt(minPixels / (height * width));
+    hBar = ceilByFactor(height * beta, factor);
+    wBar = ceilByFactor(width * beta, factor);
+  }
+
+  return [wBar, hBar];
+}
+
+export function actionParser(params: {
+  prediction: string;
+  factor: number | [number, number];
+  screenContext?: {
+    width: number;
+    height: number;
+  };
+  scaleFactor?: number;
+  mode?: 'bc' | 'o1';
+  modelVer?: UITarsModelVersion;
+}): {
+  parsed: PredictionParsed[];
+} {
+  const { prediction, factor, mode, screenContext, scaleFactor, modelVer } =
+    params;
+
+  const parsed = parseActionVlm(
+    prediction,
+    Array.isArray(factor) ? factor : [factor, factor],
+    mode,
+    screenContext,
+    scaleFactor,
+    modelVer,
+  );
+
+  return {
+    parsed,
+  };
+}
+
+export function parseActionVlm(
+  text: string,
+  factors: [number, number] = [1000, 1000],
+  mode: 'bc' | 'o1' = 'bc',
+  screenContext?: {
+    width: number;
+    height: number;
+  },
+  scaleFactor?: number,
+  modelVer: UITarsModelVersion = UITarsModelVersion.V1_0,
+): PredictionParsed[] {
+  let reflection: string | null = null;
+  let thought: string | null = null;
+  let actionStr = '';
+
+  let smartResizeFactors: [number, number] | null = null;
+  if (
+    modelVer === UITarsModelVersion.V1_5 &&
+    screenContext?.height &&
+    screenContext?.width
+  ) {
+    smartResizeFactors = smartResizeForV15(
+      screenContext.height,
+      screenContext.width,
+    );
+  }
+
+  text = text.trim();
+  if (mode === 'bc') {
+    if (text.includes('Thought:')) {
+      const thoughtMatch = text.match(
+        /Thought: ([\s\S]+?)(?=\s*Action[:：]|$)/,
+      );
+
+      if (thoughtMatch) {
+        thought = thoughtMatch[1].trim();
+      }
+    } else if (text.startsWith('Reflection:')) {
+      const reflectionMatch = text.match(
+        /Reflection: ([\s\S]+?)Action_Summary: ([\s\S]+?)(?=\s*Action[:：]|$)/,
+      );
+      if (reflectionMatch) {
+        thought = reflectionMatch[2].trim();
+        reflection = reflectionMatch[1].trim();
+      }
+    } else if (text.startsWith('Action_Summary:')) {
+      const summaryMatch = text.match(
+        /Action_Summary: (.+?)(?=\s*Action[:：]|$)/,
+      );
+      if (summaryMatch) {
+        thought = summaryMatch[1].trim();
+      }
+    }
+
+    if (!['Action:', 'Action：'].some((keyword) => text.includes(keyword))) {
+      actionStr = text;
+    } else {
+      const actionParts = text.split(/Action[:：]/);
+      actionStr = actionParts[actionParts.length - 1];
+    }
+  } else if (mode === 'o1') {
+    const thoughtMatch = text.match(/<Thought>\s*(.*?)\s*<\/Thought>/);
+    const actionSummaryMatch = text.match(
+      /\nAction_Summary:\s*(.*?)\s*Action:/,
+    );
+    const actionMatch = text.match(/\nAction:\s*(.*?)\s*<\/Output>/);
+
+    const thoughtContent = thoughtMatch ? thoughtMatch[1] : null;
+    const actionSummaryContent = actionSummaryMatch
+      ? actionSummaryMatch[1]
+      : null;
+    const actionContent = actionMatch ? actionMatch[1] : null;
+
+    thought = `${thoughtContent}\n<Action_Summary>\n${actionSummaryContent}`;
+    actionStr = actionContent || '';
+  }
+
+  const allActions = actionStr.split('\n\n');
+  const actions: PredictionParsed[] = [];
+
+  for (const rawStr of allActions) {
+    const actionInstance = parseAction(rawStr.replace(/\n/g, String.raw`\n`).trimStart());
+    let actionType = '';
+    let actionInputs: ActionInputs = {};
+
+    if (actionInstance) {
+      actionType = actionInstance.function;
+      const params = actionInstance.args;
+      actionInputs = {};
+
+      for (const [paramName, param] of Object.entries(params)) {
+        if (!param) continue;
+        const trimmedParam = (param as string).trim();
+
+        if (paramName.includes('start_box') || paramName.includes('end_box')) {
+          const oriBox = trimmedParam;
+          const numbers = oriBox
+            .replace(/[()[\]]/g, '')
+            .split(',')
+            .filter((ori) => ori !== '');
+
+          const floatNumbers = numbers.map((num, idx) => {
+            const factorIndex = idx % 2;
+            if (modelVer === UITarsModelVersion.V1_5 && smartResizeFactors) {
+              return Number.parseFloat(num) / smartResizeFactors[factorIndex];
+            }
+            return Number.parseFloat(num) / factors[factorIndex];
+          });
+
+          if (floatNumbers.length === 2) {
+            floatNumbers.push(floatNumbers[0], floatNumbers[1]);
+          }
+
+          actionInputs[
+            paramName.trim() as keyof Omit<
+              ActionInputs,
+              'start_coords' | 'end_coords'
+            >
+          ] = JSON.stringify(floatNumbers);
+
+          if (screenContext?.width && screenContext?.height) {
+            const boxKey = paramName.includes('start_box')
+              ? 'start_coords'
+              : 'end_coords';
+            const [x1, y1, x2 = x1, y2 = y1] = floatNumbers;
+            const [widthFactor, heightFactor] = factors;
+
+            actionInputs[boxKey] = [x1, y1, x2, y2].every(isNumber)
+              ? [
+                  (Math.round(
+                    ((x1 + x2) / 2) * screenContext?.width * widthFactor,
+                  ) /
+                    widthFactor) *
+                    (scaleFactor ?? 1),
+                  (Math.round(
+                    ((y1 + y2) / 2) * screenContext?.height * heightFactor,
+                  ) /
+                    heightFactor) *
+                    (scaleFactor ?? 1),
+                ]
+              : [];
+          }
+        } else {
+          actionInputs[
+            paramName.trim() as keyof Omit<
+              ActionInputs,
+              'start_coords' | 'end_coords'
+            >
+          ] = trimmedParam;
+        }
+      }
+    }
+
+    actions.push({
+      reflection: reflection,
+      thought: thought || '',
+      action_type: actionType,
+      action_inputs: actionInputs,
+    });
+  }
+
+  return actions;
+}
+
+function parseAction(actionStr: string) {
+  try {
+    actionStr = actionStr.replace(/<\|box_start\|>|<\|box_end\|>/g, '');
+
+    actionStr = actionStr
+      .replace(/(?<!start_|end_)point=/g, 'start_box=')
+      .replace(/start_point=/g, 'start_box=')
+      .replace(/end_point=/g, 'end_box=');
+
+    const functionPattern = /^(\w+)\((.*)\)$/;
+    const match = actionStr.trim().match(functionPattern);
+
+    if (!match) {
+      throw new Error('Not a function call');
+    }
+
+    const [_, functionName, argsStr] = match;
+
+    const kwargs = {};
+
+    if (argsStr.trim()) {
+      const argPairs = argsStr.match(/([^,']|'[^']*')+/g) || [];
+
+      for (const pair of argPairs) {
+        const [key, ...valueParts] = pair.split('=');
+        if (!key) continue;
+
+        let value = valueParts
+          .join('=')
+          .trim()
+          .replace(/^['"]|['"]$/g, '');
+
+        if (value.includes('<bbox>')) {
+          value = value.replace(/<bbox>|<\/bbox>/g, '').replace(/\s+/g, ',');
+          value = `(${value})`;
+        }
+
+        if (value.includes('<point>')) {
+          value = value.replace(/<point>|<\/point>/g, '').replace(/\s+/g, ',');
+          value = `(${value})`;
+        }
+
+        //@ts-ignore
+        kwargs[key.trim()] = value;
+      }
+    }
+
+    return {
+      function: functionName,
+      args: kwargs,
+    };
+  } catch (e) {
+    console.warn(`[ActionParser] Skipping invalid action: '${actionStr}'`);
+    return null;
+  }
+}

package/src/gui-subagent/action-parser/constants.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export const IMAGE_FACTOR = 28;
+export const DEFAULT_FACTOR = 1000;
+export const MIN_PIXELS = 100 * IMAGE_FACTOR * IMAGE_FACTOR;
+export const MAX_PIXELS_V1_0 = 2700 * IMAGE_FACTOR * IMAGE_FACTOR;
+export const MAX_PIXELS_DOUBAO = 5120 * IMAGE_FACTOR * IMAGE_FACTOR;
+export const MAX_PIXELS_V1_5 = 16384 * IMAGE_FACTOR * IMAGE_FACTOR;
+export const MAX_RATIO = 200;

package/src/gui-subagent/action-parser/index.ts

@@ -0,0 +1,6 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export { actionParser } from './actionParser.js';
+export type { PredictionParsed, ActionInputs, UITarsModelVersion } from './types.js';

package/src/gui-subagent/action-parser/types.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export type Coords = [number, number] | [];
+
+export type ActionInputs = Partial<{
+  content: string;
+  start_box: string;
+  end_box: string;
+  key: string;
+  hotkey: string;
+  direction: string;
+  start_coords: Coords;
+  end_coords: Coords;
+}>;
+
+export interface PredictionParsed {
+  action_inputs: ActionInputs;
+  reflection: string | null;
+  action_type: string;
+  thought: string;
+}
+
+export enum UITarsModelVersion {
+  V1_0 = '1.0',
+  V1_5 = '1.5',
+  DOUBAO_1_5_15B = 'doubao-1.5-15B',
+  DOUBAO_1_5_20B = 'doubao-1.5-20B',
+}