@aiscene/shared 8.0.2 → 8.0.4

package/dist/es/agent-tools/agent-behavior-init-args.mjs

@@ -0,0 +1,44 @@
+import { z } from "zod";
+const agentBehaviorInitArgShape = {
+    aiActContext: z.string().optional().describe('Background knowledge passed to aiAct. Default: no extra context.'),
+    replanningCycleLimit: z.number().int().nonnegative().optional().describe('Maximum number of replanning cycles for aiAct. Default: model adapter default.'),
+    waitAfterAction: z.number().nonnegative().optional().describe('Wait time in milliseconds after each action execution. Default: 300ms.'),
+    screenshotShrinkFactor: z.number().min(1).optional().describe('Screenshot shrink factor before sending images to AI. Default: 1; high values may reduce recognition quality, especially on mobile.')
+};
+function extractAgentBehaviorInitArgs(extracted) {
+    if (!extracted) return;
+    const agentOptions = {
+        ...'string' == typeof extracted.aiActContext ? {
+            aiActContext: extracted.aiActContext
+        } : {},
+        ...'string' == typeof extracted.aiActionContext ? {
+            aiActionContext: extracted.aiActionContext
+        } : {},
+        ...'number' == typeof extracted.replanningCycleLimit ? {
+            replanningCycleLimit: extracted.replanningCycleLimit
+        } : {},
+        ...'number' == typeof extracted.waitAfterAction ? {
+            waitAfterAction: extracted.waitAfterAction
+        } : {},
+        ...'number' == typeof extracted.screenshotShrinkFactor ? {
+            screenshotShrinkFactor: extracted.screenshotShrinkFactor
+        } : {}
+    };
+    return Object.keys(agentOptions).length > 0 ? agentOptions : void 0;
+}
+function stableJsonValue(value) {
+    if (Array.isArray(value)) return value.map(stableJsonValue);
+    if (value && 'object' == typeof value) return Object.fromEntries(Object.entries(value).sort(([left], [right])=>left.localeCompare(right)).map(([key, nestedValue])=>[
+            key,
+            stableJsonValue(nestedValue)
+        ]));
+    return value;
+}
+function getAgentInitArgsSignature(initArgs) {
+    if (!initArgs || 0 === Object.keys(initArgs).length) return;
+    return JSON.stringify(stableJsonValue(initArgs));
+}
+function shouldRebuildAgentForInitArgs(currentSignature, nextSignature) {
+    return currentSignature !== nextSignature && (void 0 !== currentSignature || void 0 !== nextSignature);
+}
+export { agentBehaviorInitArgShape, extractAgentBehaviorInitArgs, getAgentInitArgsSignature, shouldRebuildAgentForInitArgs };

package/dist/es/agent-tools/base-tools.mjs

@@ -0,0 +1,163 @@
+import { parseBase64 } from "@aiscene/shared/img";
+import { getDebug } from "@aiscene/shared/logger";
+import { createRecordCliCommand } from "../cli/record-command.mjs";
+import { camelToKebab, getKeyAliases } from "../key-alias-utils.mjs";
+import { generateCliReportSession, readCliReportSession, writeCliReportSession } from "./cli-report-session.mjs";
+import { createNamespacedInitArgSchema, extractNamespacedArgs, sanitizeNamespacedArgs } from "./init-arg-utils.mjs";
+import { mergeToolDefaults } from "./tool-defaults.mjs";
+import { generateCommonTools, generateToolsFromActionSpace } from "./tool-generator.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('agent-tools:base-tools');
+class BaseMidsceneTools {
+    getInitArgKeys() {
+        return this.initArgSpec ? Object.keys(this.initArgSpec.shape) : [];
+    }
+    extractAgentInitParam(args) {
+        if (!this.initArgSpec) return;
+        const extracted = extractNamespacedArgs(args, this.initArgSpec.namespace, this.getInitArgKeys());
+        if (this.initArgSpec.adapt) return this.initArgSpec.adapt(extracted);
+        return extracted;
+    }
+    sanitizeToolArgs(args) {
+        if (!this.initArgSpec) return args;
+        return sanitizeNamespacedArgs(args, this.initArgSpec.namespace, this.getInitArgKeys());
+    }
+    getAgentInitArgSchema() {
+        if (!this.initArgSpec) return {};
+        return createNamespacedInitArgSchema(this.initArgSpec.namespace, this.initArgSpec.shape);
+    }
+    getAgentInitArgCliMetadata() {
+        if (!this.initArgSpec?.cli) return;
+        const options = Object.fromEntries(this.getInitArgKeys().map((key)=>{
+            const canonicalKey = `${this.initArgSpec.namespace}.${key}`;
+            const preferredName = this.initArgSpec.cli?.preferredNames?.[key] ?? (this.initArgSpec.cli?.preferBareKeys ? camelToKebab(key) : canonicalKey);
+            const acceptedNames = new Set([
+                preferredName,
+                ...this.initArgSpec.cli?.preferBareKeys ? getKeyAliases(key) : getKeyAliases(canonicalKey)
+            ]);
+            acceptedNames.delete(preferredName);
+            return [
+                canonicalKey,
+                {
+                    preferredName,
+                    aliases: [
+                        ...acceptedNames
+                    ]
+                }
+            ];
+        }));
+        return {
+            options
+        };
+    }
+    preparePlatformTools() {
+        return [];
+    }
+    getCliReportSessionName() {}
+    createNewCliReportSession(targetIdentity) {
+        const sessionName = this.getCliReportSessionName();
+        if (!sessionName) return;
+        return generateCliReportSession(sessionName, targetIdentity);
+    }
+    commitCliReportSession(session) {
+        if (session) writeCliReportSession(session);
+    }
+    readCliReportFileName() {
+        const sessionName = this.getCliReportSessionName();
+        if (!sessionName) return;
+        return readCliReportSession(sessionName)?.reportFileName;
+    }
+    readCliReportAgentOptions() {
+        const reportFileName = this.readCliReportFileName();
+        if (!reportFileName) return;
+        return {
+            reportFileName,
+            reportAttributes: {
+                'data-group-id': reportFileName
+            }
+        };
+    }
+    async initTools() {
+        this.toolDefinitions = [];
+        const platformTools = this.preparePlatformTools();
+        this.toolDefinitions.push(...platformTools);
+        let actionSpace;
+        if (this.agent) {
+            actionSpace = await this.agent.getActionSpace();
+            debug('Action space from agent:', actionSpace.map((a)=>a.name).join(', '));
+        } else {
+            const tempDevice = this.createTemporaryDevice();
+            actionSpace = tempDevice.actionSpace();
+            await tempDevice.destroy?.();
+            debug('Action space from temporary device:', actionSpace.map((a)=>a.name).join(', '));
+        }
+        const actionTools = generateToolsFromActionSpace(actionSpace, (args = {})=>this.ensureAgent(this.extractAgentInitParam(args)), (args = {})=>this.sanitizeToolArgs(args), this.getAgentInitArgSchema(), this.getAgentInitArgCliMetadata(), this.toolDefaults);
+        const commonTools = generateCommonTools((args = {})=>this.ensureAgent(this.extractAgentInitParam(args)), this.getAgentInitArgSchema(), this.getAgentInitArgCliMetadata(), this.toolDefaults);
+        this.toolDefinitions.push(...actionTools, ...commonTools);
+        debug('Total tools prepared:', this.toolDefinitions.length);
+    }
+    async destroy() {
+        await this.agent?.destroy?.();
+    }
+    getToolDefinitions() {
+        return this.toolDefinitions;
+    }
+    getCliToolDefinitions() {
+        return [
+            createRecordCliCommand((args = {})=>this.ensureAgent(this.extractAgentInitParam(args)), this.getAgentInitArgSchema(), this.getAgentInitArgCliMetadata())
+        ];
+    }
+    setAgent(agent) {
+        this.agent = agent;
+    }
+    setToolDefaults(toolDefaults) {
+        this.toolDefaults = mergeToolDefaults(this.toolDefaults, toolDefaults);
+    }
+    buildScreenshotContent(screenshot) {
+        const { mimeType, body } = parseBase64(screenshot);
+        return [
+            {
+                type: 'image',
+                data: body,
+                mimeType
+            }
+        ];
+    }
+    buildTextResult(text) {
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text
+                }
+            ]
+        };
+    }
+    createDisconnectHandler(platformName) {
+        return async ()=>{
+            if (!this.agent) return this.buildTextResult('No active connection to disconnect');
+            try {
+                await this.agent.destroy?.();
+            } catch (error) {
+                debug('Failed to destroy agent during disconnect:', error);
+            }
+            this.agent = void 0;
+            return this.buildTextResult(`Disconnected from ${platformName}`);
+        };
+    }
+    constructor(){
+        _define_property(this, "agent", void 0);
+        _define_property(this, "toolDefinitions", []);
+        _define_property(this, "toolDefaults", {});
+    }
+}
+export { BaseMidsceneTools };

package/dist/es/agent-tools/chrome-path.mjs

@@ -0,0 +1,50 @@
+import { existsSync } from "node:fs";
+import { MIDSCENE_CHROME_PATH, MIDSCENE_MCP_CHROME_PATH, globalConfigManager } from "../env/index.mjs";
+import { getDebug } from "../logger.mjs";
+const warnChromePath = getDebug('agent-tools:chrome-path', {
+    console: true
+});
+let hasWarnedLegacyChromePath = false;
+let cachedSystemChromePath;
+function getSystemChromePath() {
+    if (void 0 !== cachedSystemChromePath) return cachedSystemChromePath;
+    const platform = process.platform;
+    const chromePaths = {
+        darwin: [
+            '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+            '/Applications/Chromium.app/Contents/MacOS/Chromium'
+        ],
+        win32: [
+            'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
+            'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
+            `C:\\Users\\${process.env.USERNAME ?? process.env.USER}\\AppData\\Local\\Google\\Chrome\\Application\\chrome.exe`
+        ],
+        linux: [
+            '/opt/google/chrome/chrome',
+            '/opt/google/chrome/google-chrome',
+            '/usr/bin/google-chrome-stable',
+            '/usr/bin/google-chrome',
+            '/usr/bin/chromium-browser',
+            '/usr/bin/chromium',
+            '/snap/bin/chromium'
+        ]
+    };
+    const paths = chromePaths[platform] ?? [];
+    const foundPath = paths.find((p)=>existsSync(p));
+    if (foundPath) cachedSystemChromePath = foundPath;
+    return foundPath;
+}
+function resolveChromePath() {
+    const primaryEnvPath = globalConfigManager.getEnvConfigValue(MIDSCENE_CHROME_PATH);
+    const legacyEnvPath = globalConfigManager.getEnvConfigValue(MIDSCENE_MCP_CHROME_PATH);
+    const envPath = primaryEnvPath || legacyEnvPath;
+    if (!primaryEnvPath && legacyEnvPath && !hasWarnedLegacyChromePath) {
+        warnChromePath('MIDSCENE_MCP_CHROME_PATH is deprecated. Use MIDSCENE_CHROME_PATH instead.');
+        hasWarnedLegacyChromePath = true;
+    }
+    if (envPath && 'auto' !== envPath && existsSync(envPath)) return envPath;
+    const systemPath = getSystemChromePath();
+    if (systemPath) return systemPath;
+    throw new Error('Chrome not found. Install Google Chrome or set MIDSCENE_CHROME_PATH environment variable.');
+}
+export { getSystemChromePath, resolveChromePath };

package/dist/es/agent-tools/cli-report-session.mjs

@@ -0,0 +1,78 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { getMidsceneRunBaseDir, getMidsceneRunSubDir } from "../common.mjs";
+const sessionDirName = 'cli-report-session';
+function sanitizeSessionName(sessionName) {
+    return sessionName.replace(/[^a-zA-Z0-9._-]/g, '_') || 'default';
+}
+function sanitizeFileSegment(segment) {
+    const sanitized = segment.replace(/[^a-zA-Z0-9._-]/g, '_') || 'unknown';
+    return sanitized.slice(0, 80);
+}
+function ensureHtmlFileName(reportFileName) {
+    return reportFileName.endsWith('.html') ? reportFileName : `${reportFileName}.html`;
+}
+function formatDateForFileName(date) {
+    const pad = (value)=>String(value).padStart(2, '0');
+    const day = [
+        date.getFullYear(),
+        pad(date.getMonth() + 1),
+        pad(date.getDate())
+    ].join('-');
+    const time = [
+        pad(date.getHours()),
+        pad(date.getMinutes()),
+        pad(date.getSeconds())
+    ].join('-');
+    return `${day}_${time}`;
+}
+function randomId() {
+    return Math.random().toString(36).slice(2, 10);
+}
+function getCliReportSessionDir() {
+    const dir = join(getMidsceneRunBaseDir(), sessionDirName);
+    if (!existsSync(dir)) mkdirSync(dir, {
+        recursive: true
+    });
+    return dir;
+}
+function getCliReportSessionPath(sessionName) {
+    return join(getCliReportSessionDir(), `${sanitizeSessionName(sessionName)}.json`);
+}
+function generateCliReportSession(sessionName, targetIdentity) {
+    const identitySegment = targetIdentity ? `-${sanitizeFileSegment(targetIdentity)}` : '';
+    const reportFileName = `${sanitizeSessionName(sessionName)}${identitySegment}-${formatDateForFileName(new Date())}-${randomId()}`;
+    const reportPath = join(getMidsceneRunSubDir('report'), ensureHtmlFileName(reportFileName));
+    const session = {
+        version: 1,
+        sessionName,
+        ...targetIdentity ? {
+            targetIdentity
+        } : {},
+        reportFileName,
+        reportPath,
+        createdAt: Date.now()
+    };
+    return session;
+}
+function writeCliReportSession(session) {
+    writeFileSync(getCliReportSessionPath(session.sessionName), JSON.stringify(session, null, 2), 'utf-8');
+}
+function createCliReportSession(sessionName, targetIdentity) {
+    const session = generateCliReportSession(sessionName, targetIdentity);
+    writeCliReportSession(session);
+    return session;
+}
+function readCliReportSession(sessionName) {
+    const sessionPath = getCliReportSessionPath(sessionName);
+    if (!existsSync(sessionPath)) return;
+    try {
+        const raw = readFileSync(sessionPath, 'utf-8');
+        const parsed = JSON.parse(raw);
+        if (1 !== parsed.version || parsed.sessionName !== sessionName || 'string' != typeof parsed.reportFileName || !parsed.reportFileName.trim() || /[\\/]/.test(parsed.reportFileName)) return;
+        return parsed;
+    } catch  {
+        return;
+    }
+}
+export { createCliReportSession, generateCliReportSession, readCliReportSession, writeCliReportSession };

package/dist/es/agent-tools/error-formatter.mjs

@@ -0,0 +1,106 @@
+const maxSerializedStringLength = 4096;
+const truncatedStringSuffix = '… [truncated]';
+function isObject(error) {
+    return 'object' == typeof error && null !== error || 'function' == typeof error;
+}
+function safelyReadProperty(error, key) {
+    try {
+        return Reflect.get(error, key);
+    } catch  {
+        return;
+    }
+}
+function readBoundedDiagnostic(error, keys) {
+    for (const key of keys){
+        const value = safelyReadProperty(error, key);
+        if ('number' == typeof value) return value;
+        if ('string' == typeof value) return truncateSerializedErrorString(value);
+    }
+}
+function truncateSerializedErrorString(value) {
+    if (value.length <= maxSerializedStringLength) return value;
+    return `${value.slice(0, maxSerializedStringLength - truncatedStringSuffix.length)}${truncatedStringSuffix}`;
+}
+function readNonEmptyMessage(error) {
+    const message = safelyReadProperty(error, 'message');
+    return 'string' == typeof message && message ? message : void 0;
+}
+function extractStringMessage(error) {
+    const directMessage = readNonEmptyMessage(error);
+    if (directMessage) return directMessage;
+    for (const nestedKey of [
+        'error',
+        'cause'
+    ]){
+        const nestedError = safelyReadProperty(error, nestedKey);
+        if (!isObject(nestedError)) continue;
+        const nestedMessage = readNonEmptyMessage(nestedError);
+        if (nestedMessage) return nestedMessage;
+    }
+}
+function getErrorMessage(error) {
+    const result = serializeErrorValue(error);
+    const serialized = result.error;
+    if (!isObject(error) || result.hasMessage) return serialized.message;
+    const summary = {
+        name: serialized.name,
+        message: serialized.message
+    };
+    for (const key of [
+        'code',
+        'status',
+        'requestId'
+    ]){
+        const value = serialized[key];
+        if (void 0 !== value) summary[key] = value;
+    }
+    return truncateSerializedErrorString(JSON.stringify(summary));
+}
+function getErrorStack(error) {
+    if (!isObject(error)) return;
+    const stack = safelyReadProperty(error, 'stack');
+    return 'string' == typeof stack ? stack : void 0;
+}
+function serializeErrorValue(error) {
+    if (!isObject(error)) {
+        const message = truncateSerializedErrorString(String(error));
+        return {
+            error: {
+                name: 'NonError',
+                message: message.trim() ? message : 'Empty string thrown'
+            },
+            hasMessage: Boolean(message.trim())
+        };
+    }
+    const name = safelyReadProperty(error, 'name');
+    const serializedName = truncateSerializedErrorString('string' == typeof name && name ? name : 'Error');
+    const message = extractStringMessage(error);
+    const serialized = {
+        name: serializedName,
+        message: truncateSerializedErrorString(message ?? `${serializedName} without a message`)
+    };
+    const stack = getErrorStack(error);
+    if (stack) serialized.stack = truncateSerializedErrorString(stack);
+    const code = readBoundedDiagnostic(error, [
+        'code'
+    ]);
+    const status = readBoundedDiagnostic(error, [
+        'status',
+        'statusCode'
+    ]);
+    const requestId = readBoundedDiagnostic(error, [
+        'requestId',
+        'requestID'
+    ]);
+    if (void 0 !== code) serialized.code = code;
+    if (void 0 !== status) serialized.status = status;
+    if (void 0 !== requestId) serialized.requestId = requestId;
+    return {
+        error: serialized,
+        hasMessage: void 0 !== message
+    };
+}
+function serializeError(error) {
+    return serializeErrorValue(error).error;
+}
+export { getErrorMessage, getErrorStack, serializeError, truncateSerializedErrorString };

package/dist/es/agent-tools/index.mjs

@@ -0,0 +1,9 @@
+export * from "./base-tools.mjs";
+export * from "./tool-defaults.mjs";
+export * from "./agent-behavior-init-args.mjs";
+export * from "./init-arg-utils.mjs";
+export * from "./error-formatter.mjs";
+export * from "./tool-generator.mjs";
+export * from "./types.mjs";
+export * from "./chrome-path.mjs";
+export * from "./observation-record.mjs";

package/dist/es/agent-tools/init-arg-utils.mjs

@@ -0,0 +1,38 @@
+import { getKeyAliases, isRecord } from "../key-alias-utils.mjs";
+function readAliasedValue(args, key) {
+    for (const alias of getKeyAliases(key))if (alias in args) return args[alias];
+}
+function readNamespacedArg(args, namespace, key) {
+    const namespacedArgs = readAliasedValue(args, namespace);
+    if (isRecord(namespacedArgs)) {
+        const nestedValue = readAliasedValue(namespacedArgs, key);
+        if (void 0 !== nestedValue) return nestedValue;
+    }
+    const dottedValue = readAliasedValue(args, `${namespace}.${key}`);
+    if (void 0 !== dottedValue) return dottedValue;
+    const directValue = readAliasedValue(args, key);
+    if (void 0 !== directValue) return directValue;
+}
+function extractNamespacedArgs(args, namespace, keys) {
+    const extracted = {};
+    for (const key of keys){
+        const value = readNamespacedArg(args, namespace, key);
+        if (void 0 !== value) extracted[key] = value;
+    }
+    return Object.keys(extracted).length > 0 ? extracted : void 0;
+}
+function sanitizeNamespacedArgs(args, namespace, keys) {
+    const excludedKeys = new Set(getKeyAliases(namespace));
+    for (const key of keys){
+        for (const alias of getKeyAliases(key))excludedKeys.add(alias);
+        for (const alias of getKeyAliases(`${namespace}.${key}`))excludedKeys.add(alias);
+    }
+    return Object.fromEntries(Object.entries(args).filter(([key])=>!excludedKeys.has(key)));
+}
+function createNamespacedInitArgSchema(namespace, shape) {
+    return Object.fromEntries(Object.entries(shape).map(([key, value])=>[
+            `${namespace}.${key}`,
+            value
+        ]));
+}
+export { createNamespacedInitArgSchema, extractNamespacedArgs, sanitizeNamespacedArgs };

package/dist/es/agent-tools/observation-artifact.mjs

@@ -0,0 +1,5 @@
+const observationArtifactAdapterSymbol = Symbol('midscene.observationArtifactAdapter');
+function resolveObservationArtifactAdapter(agent) {
+    return agent[observationArtifactAdapterSymbol];
+}
+export { observationArtifactAdapterSymbol, resolveObservationArtifactAdapter };