@aiscene/core 1.7.0

package/dist/types/yaml/builder.d.ts

@@ -0,0 +1,2 @@
+import type { MidsceneYamlScriptWebEnv, MidsceneYamlTask } from '../types';
+export declare function buildYaml(env: MidsceneYamlScriptWebEnv, tasks: MidsceneYamlTask[]): string;

package/dist/types/yaml/index.d.ts

@@ -0,0 +1,4 @@
+export * from '../yaml';
+export * from './player';
+export * from './builder';
+export * from './utils';

package/dist/types/yaml/player.d.ts

@@ -0,0 +1,34 @@
+import type { Agent } from '../agent/agent';
+import type { FreeFn, MidsceneYamlScript, MidsceneYamlScriptEnv, ScriptPlayerStatusValue, ScriptPlayerTaskStatus } from '../types';
+export declare class ScriptPlayer<T extends MidsceneYamlScriptEnv> {
+    private script;
+    private setupAgent;
+    onTaskStatusChange?: ((taskStatus: ScriptPlayerTaskStatus) => void) | undefined;
+    currentTaskIndex?: number;
+    taskStatusList: ScriptPlayerTaskStatus[];
+    status: ScriptPlayerStatusValue;
+    reportFile?: string | null;
+    result: Record<string, any>;
+    private unnamedResultIndex;
+    output?: string | null;
+    unstableLogContent?: string | null;
+    errorInSetup?: Error;
+    private interfaceAgent;
+    agentStatusTip?: string;
+    target?: MidsceneYamlScriptEnv;
+    private actionSpace;
+    private scriptPath?;
+    constructor(script: MidsceneYamlScript, setupAgent: (platform: T) => Promise<{
+        agent: Agent;
+        freeFn: FreeFn[];
+    }>, onTaskStatusChange?: ((taskStatus: ScriptPlayerTaskStatus) => void) | undefined, scriptPath?: string);
+    private setResult;
+    private setPlayerStatus;
+    private notifyCurrentTaskStatusChange;
+    private setTaskStatus;
+    private setTaskIndex;
+    private flushResult;
+    private flushUnstableLogContent;
+    playTask(taskStatus: ScriptPlayerTaskStatus, agent: Agent): Promise<void>;
+    run(): Promise<void>;
+}

package/dist/types/yaml/utils.d.ts

@@ -0,0 +1,9 @@
+import type { TUserPrompt } from '../common';
+import type { DetailedLocateParam, LocateOption, MidsceneYamlScript } from '../types';
+export declare function interpolateEnvVars(content: string): string;
+export declare function parseYamlScript(content: string, filePath?: string): MidsceneYamlScript;
+export declare function buildDetailedLocateParam(locatePrompt: TUserPrompt, opt?: LocateOption): DetailedLocateParam | undefined;
+export declare function buildDetailedLocateParamAndRestParams(locatePrompt: TUserPrompt, opt: LocateOption | undefined, excludeKeys?: string[]): {
+    locateParam: DetailedLocateParam | undefined;
+    restParams: Record<string, any>;
+};

package/dist/types/yaml.d.ts

@@ -0,0 +1,217 @@
+import type { TMultimodalPrompt, TUserPrompt } from './common';
+import type { AndroidDeviceOpt, IOSDeviceOpt } from './device';
+import type { AgentOpt, LocateResultElement } from './types';
+import type { UIContext } from './types';
+export interface LocateOption extends Partial<TMultimodalPrompt> {
+    prompt?: TUserPrompt;
+    deepLocate?: boolean;
+    /** @deprecated Use `deepLocate` instead. Kept for backward compatibility. */
+    deepThink?: boolean;
+    cacheable?: boolean;
+    xpath?: string;
+    uiContext?: UIContext;
+    fileChooserAccept?: string | string[];
+}
+export interface ServiceExtractOption {
+    domIncluded?: boolean | 'visible-only';
+    screenshotIncluded?: boolean;
+    /** 当为 true 时，强制 aiBoolean 只根据截图判断，不过度联想值判断错误 */
+    strictBoolean?: boolean;
+    [key: string]: unknown;
+}
+export interface DetailedLocateParam extends Omit<LocateOption, 'deepThink' | keyof TMultimodalPrompt> {
+    prompt: TUserPrompt;
+}
+export type ScrollType = 'singleAction' | 'scrollToBottom' | 'scrollToTop' | 'scrollToRight' | 'scrollToLeft' | 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';
+export type ActionScrollParam = {
+    direction?: 'down' | 'up' | 'right' | 'left';
+    scrollType?: ScrollType;
+    distance?: number | null;
+    locate?: LocateResultElement;
+};
+export type ScrollParam = Omit<ActionScrollParam, 'locate'>;
+export interface MidsceneYamlScript {
+    target?: MidsceneYamlScriptWebEnv;
+    web?: MidsceneYamlScriptWebEnv;
+    android?: MidsceneYamlScriptAndroidEnv;
+    ios?: MidsceneYamlScriptIOSEnv;
+    computer?: MidsceneYamlScriptComputerEnv;
+    interface?: MidsceneYamlScriptEnvGeneralInterface;
+    config?: MidsceneYamlScriptConfig;
+    agent?: MidsceneYamlScriptAgentOpt;
+    tasks: MidsceneYamlTask[];
+}
+export interface MidsceneYamlTask {
+    name: string;
+    flow: MidsceneYamlFlowItem[];
+    continueOnError?: boolean;
+}
+/**
+ * Agent configuration options that can be specified in YAML scripts.
+ *
+ * This type includes serializable fields from AgentOpt, excluding non-serializable
+ * fields like functions and complex objects. All fields are optional.
+ *
+ * @remarks
+ * - testId priority: CLI parameter > YAML agent.testId > filename
+ * - These settings apply to all platforms (Web, Android, iOS, Generic Interface)
+ * - modelConfig is configured through environment variables, not in YAML
+ *
+ * @example
+ * ```yaml
+ * agent:
+ *   testId: "checkout-test"
+ *   groupName: "E2E Test Suite"
+ *   generateReport: true
+ *   replanningCycleLimit: 30
+ *   cache:
+ *     id: "checkout-cache"
+ *     strategy: "read-write"
+ * ```
+ */
+export type MidsceneYamlScriptAgentOpt = Pick<AgentOpt, 'testId' | 'groupName' | 'groupDescription' | 'generateReport' | 'autoPrintReportMsg' | 'reportFileName' | 'replanningCycleLimit' | 'aiActContext' | 'aiActionContext' | 'cache' | 'screenshotShrinkFactor'>;
+export interface MidsceneYamlScriptConfig {
+    output?: string;
+    unstableLogContent?: boolean | string;
+}
+export interface MidsceneYamlScriptEnvGeneralInterface {
+    module: string;
+    export?: string;
+    param?: Record<string, any>;
+}
+export interface MidsceneYamlScriptWebEnv extends MidsceneYamlScriptConfig, MidsceneYamlScriptAgentOpt {
+    serve?: string;
+    url: string;
+    userAgent?: string;
+    acceptInsecureCerts?: boolean;
+    viewportWidth?: number;
+    viewportHeight?: number;
+    deviceScaleFactor?: number;
+    waitForNetworkIdle?: {
+        timeout?: number;
+        continueOnNetworkIdleError?: boolean;
+    };
+    cookie?: string;
+    forceSameTabNavigation?: boolean;
+    /**
+     * Custom Chrome launch arguments (Puppeteer only, not supported in bridge mode).
+     *
+     * Allows passing custom command-line arguments to Chrome/Chromium when launching the browser.
+     * This is useful for testing scenarios that require specific browser configurations.
+     *
+     * ⚠️ Security Warning: Some arguments (e.g., --no-sandbox, --disable-web-security) may
+     * reduce browser security. Use only in controlled testing environments.
+     *
+     * @example
+     * ```yaml
+     * web:
+     *   url: https://example.com
+     *   chromeArgs:
+     *     - '--disable-features=ThirdPartyCookiePhaseout'
+     *     - '--disable-features=SameSiteByDefaultCookies'
+     *     - '--window-size=1920,1080'
+     * ```
+     */
+    chromeArgs?: string[];
+    bridgeMode?: false | 'newTabWithUrl' | 'currentTab';
+    closeNewTabsAfterDisconnect?: boolean;
+    /**
+     * CDP (Chrome DevTools Protocol) endpoint URL.
+     * When specified, connects to an existing Chrome browser via CDP instead of launching a new one.
+     *
+     * @example
+     * ```yaml
+     * web:
+     *   url: https://example.com
+     *   cdpEndpoint: ws://localhost:9222/devtools/browser/xxxx
+     * ```
+     */
+    cdpEndpoint?: string;
+}
+export interface MidsceneYamlScriptAndroidEnv extends MidsceneYamlScriptConfig, Omit<AndroidDeviceOpt, 'customActions'> {
+    deviceId?: string;
+    launch?: string;
+}
+export interface MidsceneYamlScriptIOSEnv extends MidsceneYamlScriptConfig, Omit<IOSDeviceOpt, 'customActions'> {
+    launch?: string;
+}
+export interface MidsceneYamlScriptComputerEnv extends MidsceneYamlScriptConfig {
+    displayId?: string;
+}
+export type MidsceneYamlScriptEnv = MidsceneYamlScriptWebEnv | MidsceneYamlScriptAndroidEnv | MidsceneYamlScriptIOSEnv | MidsceneYamlScriptComputerEnv;
+export interface MidsceneYamlFlowItemAIAction {
+    aiAction?: string;
+    ai?: string;
+    aiAct?: string;
+    aiActionProgressTips?: string[];
+    cacheable?: boolean;
+    [key: string]: unknown;
+}
+export interface MidsceneYamlFlowItemAIAssert extends ServiceExtractOption {
+    aiAssert: string;
+    errorMessage?: string;
+    name?: string;
+}
+export interface MidsceneYamlFlowItemAIWaitFor extends ServiceExtractOption {
+    aiWaitFor: string;
+    timeout?: number;
+}
+export interface MidsceneYamlFlowItemEvaluateJavaScript {
+    javascript: string;
+    name?: string;
+}
+export interface MidsceneYamlFlowItemSleep {
+    sleep: number;
+}
+export interface MidsceneYamlFlowItemLogScreenshot {
+    logScreenshot?: string;
+    recordToReport?: string;
+    content?: string;
+}
+export type MidsceneYamlFlowItem = MidsceneYamlFlowItemAIAction | MidsceneYamlFlowItemAIAssert | MidsceneYamlFlowItemAIWaitFor | MidsceneYamlFlowItemEvaluateJavaScript | MidsceneYamlFlowItemSleep | MidsceneYamlFlowItemLogScreenshot;
+export interface FreeFn {
+    name: string;
+    fn: () => void;
+}
+export interface ScriptPlayerTaskStatus extends MidsceneYamlTask {
+    status: ScriptPlayerStatusValue;
+    currentStep?: number;
+    totalSteps: number;
+    error?: Error;
+}
+export type ScriptPlayerStatusValue = 'init' | 'running' | 'done' | 'error';
+export interface MidsceneYamlConfig {
+    concurrent?: number;
+    continueOnError?: boolean;
+    summary?: string;
+    shareBrowserContext?: boolean;
+    web?: MidsceneYamlScriptWebEnv;
+    android?: MidsceneYamlScriptAndroidEnv;
+    ios?: MidsceneYamlScriptIOSEnv;
+    files: string[];
+    headed?: boolean;
+    keepWindow?: boolean;
+    dotenvOverride?: boolean;
+    dotenvDebug?: boolean;
+}
+export interface MidsceneYamlConfigOutput {
+    format?: 'json';
+    path?: string;
+}
+export interface MidsceneYamlConfigResult {
+    file: string;
+    success: boolean;
+    executed: boolean;
+    output?: string | null;
+    report?: string | null;
+    error?: string;
+    duration?: number;
+    /**
+     * Type of result:
+     * - 'success': All tasks completed successfully
+     * - 'failed': Execution failed (player error)
+     * - 'partialFailed': Some tasks failed but execution continued (continueOnError)
+     * - 'notExecuted': Not executed due to previous failures
+     */
+    resultType?: 'success' | 'failed' | 'partialFailed' | 'notExecuted';
+}

package/package.json

@@ -0,0 +1,130 @@
+{
+  "name": "@aiscene/core",
+  "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
+  "version": "1.7.0",
+  "repository": "https://github.com/web-infra-dev/midscene",
+  "homepage": "https://midscenejs.com/",
+  "main": "./dist/lib/index.js",
+  "types": "./dist/types/index.d.ts",
+  "module": "./dist/es/index.mjs",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/es/index.mjs",
+      "require": "./dist/lib/index.js"
+    },
+    "./utils": {
+      "types": "./dist/types/utils.d.ts",
+      "import": "./dist/es/utils.mjs",
+      "require": "./dist/lib/utils.js"
+    },
+    "./ai-model": {
+      "types": "./dist/types/ai-model/index.d.ts",
+      "import": "./dist/es/ai-model/index.mjs",
+      "require": "./dist/lib/ai-model/index.js"
+    },
+    "./tree": {
+      "types": "./dist/types/tree.d.ts",
+      "import": "./dist/es/tree.mjs",
+      "require": "./dist/lib/tree.js"
+    },
+    "./device": {
+      "types": "./dist/types/device/index.d.ts",
+      "import": "./dist/es/device/index.mjs",
+      "require": "./dist/lib/device/index.js"
+    },
+    "./agent": {
+      "types": "./dist/types/agent/index.d.ts",
+      "import": "./dist/es/agent/index.mjs",
+      "require": "./dist/lib/agent/index.js"
+    },
+    "./yaml": {
+      "types": "./dist/types/yaml/index.d.ts",
+      "import": "./dist/es/yaml/index.mjs",
+      "require": "./dist/lib/yaml/index.js"
+    },
+    "./report": {
+      "types": "./dist/types/report.d.ts",
+      "import": "./dist/es/report.mjs",
+      "require": "./dist/lib/report.js"
+    },
+    "./skill": {
+      "types": "./dist/types/skill/index.d.ts",
+      "import": "./dist/es/skill/index.mjs",
+      "require": "./dist/lib/skill/index.js"
+    }
+  },
+  "typesVersions": {
+    "*": {
+      ".": [
+        "./dist/types/index.d.ts"
+      ],
+      "utils": [
+        "./dist/types/utils.d.ts"
+      ],
+      "ai-model": [
+        "./dist/types/ai-model.d.ts"
+      ],
+      "tree": [
+        "./dist/types/tree.d.ts"
+      ],
+      "device": [
+        "./dist/types/device.d.ts"
+      ],
+      "agent": [
+        "./dist/types/agent.d.ts"
+      ],
+      "yaml": [
+        "./dist/types/yaml.d.ts"
+      ],
+      "mcp": [
+        "./dist/types/skill/index.d.ts"
+      ]
+    }
+  },
+  "dependencies": {
+    "@ui-tars/action-parser": "1.2.3",
+    "dayjs": "^1.11.11",
+    "dotenv": "^16.4.5",
+    "fetch-socks": "^1.3.0",
+    "js-yaml": "4.1.0",
+    "jsonrepair": "3.12.0",
+    "openai": "6.3.0",
+    "semver": "7.5.2",
+    "undici": "^6.0.0",
+    "zod": "3.24.3",
+    "@midscene/shared": "1.6.0"
+  },
+  "devDependencies": {
+    "@rslib/core": "^0.18.3",
+    "@types/js-yaml": "4.0.9",
+    "@types/node": "^18.0.0",
+    "@types/node-fetch": "2.6.11",
+    "@types/semver": "7.7.0",
+    "langsmith": "^0.3.74",
+    "typescript": "^5.8.3",
+    "vitest": "3.0.5"
+  },
+  "engines": {
+    "node": ">=18.19.0"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "license": "MIT",
+  "scripts": {
+    "dev": "npm run build:watch",
+    "build": "rslib build",
+    "build:watch": "USE_DEV_REPORT=1 rslib build --watch --no-clean",
+    "test": "vitest --run",
+    "test:u": "vitest --run -u",
+    "test:ai": "AITEST=true npm run test",
+    "computer": "TEST_COMPUTER=true npm run test:ai -- tests/ai/evaluate/computer.test.ts",
+    "test:parse-action": "npm run test:ai -- tests/ai/parse-action.test.ts"
+  }
+}