newmark-agent 0.4.7 → 0.4.9

package/dist/main.js

@@ -120,6 +120,7 @@ let _forceQuit = false;
 let forcedExitTimer = null;
 let electronBrowserUseHost = null;
 let browserUseEngine = null;
+const mobileServerWorkEventSubscribers = new Set();
 // A single renderer can host several hidden conversation-bound guests. The
 // legacy host map is retained for focused-window discovery, while this map is
 // the authoritative Browser-Use/right-sidebar binding.
@@ -288,7 +289,7 @@ async function resetWslAgentClient() {
     if (wslAgentClient)
         await wslAgentClient.resetAgent();
 }
-function broadcastAgentWorkEvent(event) {
+function broadcastAgentWorkEvent(event, mirrorToMobile = true) {
     const workEvent = event;
     if ((workEvent.type === 'done' || workEvent.type === 'error') && workEvent.runtimeKey) {
         browserUseEngine?.clearRuntime(workEvent.runtimeKey);
@@ -299,6 +300,14 @@ function broadcastAgentWorkEvent(event) {
             continue;
         win.webContents.send('agent:workEvent', event);
     }
+    if (mirrorToMobile) {
+        for (const listener of mobileServerWorkEventSubscribers) {
+            try {
+                listener(event);
+            }
+            catch { /* ignore disconnected mobile bridge */ }
+        }
+    }
 }
 function ensureConversationKernel(root) {
     if (!agent)
@@ -1882,16 +1891,195 @@ else {
                     ensureWorkspaceRegistryWatcher();
                     restoreStoredFlowSuspension();
                     recordStartup('agent-ready');
-                    // 远程触及开关开启 → GUI 进程内托管启动 mobile server（托盘常驻不中断）
-                    if (agent.config.getBool('remote', 'touch_enabled')) {
-                        try {
-                            const { runServer } = require('./server');
+                    // GUI 始终注册 mobile server 的运行时桥接；开关打开时才开始监听。
+                    // 这样从关闭态切到开启态时也能在当前 GUI 进程中原地重启服务。
+                    try {
+                        const { configureHostedServer, runServer } = require('./server');
+                        configureHostedServer(root, {
+                            agent,
+                            automation,
+                            onWorkEvent: event => broadcastAgentWorkEvent(event, false),
+                            subscribeWorkEvents: listener => {
+                                mobileServerWorkEventSubscribers.add(listener);
+                                return () => mobileServerWorkEventSubscribers.delete(listener);
+                            },
+                            conversationUiState: async (requested) => {
+                                const target = conversationRuntimeTarget(requested);
+                                const snapshot = wslBackendEnabled()
+                                    ? await ensureWslConversationPool().snapshot(target)
+                                    : await ensureElectronUtilityPool().snapshot(target);
+                                const flowRunning = flowRunningForTarget(target);
+                                const flowSuspension = flowSuspensionForTarget(target);
+                                return {
+                                    ...snapshot,
+                                    flow: flowRunning ? {
+                                        running: true,
+                                        paused: false,
+                                        name: flowRunning.name,
+                                        promptText: String(activeFlowStateFor(target)?.input || ''),
+                                        message: '',
+                                    } : flowSuspension ? {
+                                        running: true,
+                                        paused: true,
+                                        name: flowSuspension.workflowName || '',
+                                        promptText: String(flowSuspension.input || ''),
+                                        message: String(flowSuspension.message || ''),
+                                        reason: flowSuspension.reason || '',
+                                    } : null,
+                                };
+                            },
+                            conversationUiAction: async (requested, action, value, input) => {
+                                const target = conversationRuntimeTarget(requested);
+                                if (action.startsWith('queue_')) {
+                                    const queueAction = action === 'queue_enqueue' ? 'enqueue'
+                                        : action === 'queue_update' ? 'update'
+                                            : action === 'queue_delete' ? 'delete'
+                                                : action === 'queue_reorder' ? 'reorder'
+                                                    : action === 'queue_toggle_pause' ? 'toggle_pause'
+                                                        : 'guide';
+                                    const queueInput = {
+                                        id: String(input?.id || ''),
+                                        text: String(input?.text || value || ''),
+                                        requestedMode: String(input?.requestedMode || 'build'),
+                                        goalObjective: String(input?.goalObjective || ''),
+                                        createdAt: String(input?.createdAt || ''),
+                                        orderedIds: Array.isArray(input?.orderedIds)
+                                            ? input.orderedIds.map((id) => String(id || ''))
+                                            : undefined,
+                                    };
+                                    return wslBackendEnabled()
+                                        ? await ensureWslConversationPool().queueAction(target, queueAction, queueInput)
+                                        : await ensureElectronUtilityPool().queueAction(target, queueAction, queueInput);
+                                }
+                                if (action === 'goal_update') {
+                                    return { goal: await mutateTargetConversation(target, () => {
+                                            const isolated = isolatedConversationAgent(target);
+                                            isolated.updateGoal(String(value || '').trim());
+                                            isolated.setMode('goal');
+                                            isolated.saveWorkspaceConversationState(true);
+                                            return isolated.getConversationSnapshot(target.conversationId).goal;
+                                        }) };
+                                }
+                                if (action === 'goal_guide' || action === 'conversation_guide') {
+                                    const objective = action === 'goal_guide' ? String(value || '').trim() : '';
+                                    const guideText = String(value || '').trim();
+                                    const snapshot = wslBackendEnabled()
+                                        ? await ensureWslConversationPool().snapshot(target)
+                                        : await ensureElectronUtilityPool().snapshot(target);
+                                    const runtime = snapshot.runtime;
+                                    if (!guideText || !runtime?.running || !runtime.runId) {
+                                        return { ok: false, error: action === 'goal_guide'
+                                                ? 'There is no active Build for this Goal Guide.'
+                                                : 'Target conversation is not running.' };
+                                    }
+                                    const prefix = 'Goal for the current Build:\n';
+                                    const now = new Date().toISOString();
+                                    const envelope = {
+                                        clientMessageId: (0, crypto_1.randomUUID)(),
+                                        guideId: (0, crypto_1.randomUUID)(),
+                                        target: { workspaceId: target.workspaceId, conversationId: target.conversationId },
+                                        runId: runtime.runId,
+                                        deliveryMode: 'steer',
+                                        text: objective ? prefix + objective : guideText,
+                                        goalObjective: objective || undefined,
+                                        createdAt: now,
+                                    };
+                                    const receipt = wslBackendEnabled()
+                                        ? await ensureWslConversationPool().enqueueGuide(envelope)
+                                        : await ensureElectronUtilityPool().enqueueGuide(envelope);
+                                    return { ok: receipt.status !== 'rejected', receipt };
+                                }
+                                if (action === 'goal_toggle_pause') {
+                                    const resident = wslBackendEnabled()
+                                        ? await ensureWslConversationPool().toggleGoalPause(target)
+                                        : await ensureElectronUtilityPool().toggleGoalPause(target);
+                                    const paused = resident !== null ? resident : await mutateTargetConversation(target, () => isolatedConversationAgent(target).toggleGoalPause());
+                                    return { ok: true, paused };
+                                }
+                                if (action === 'goal_clear') {
+                                    const resident = wslBackendEnabled()
+                                        ? await ensureWslConversationPool().clearGoal(target)
+                                        : await ensureElectronUtilityPool().clearGoal(target);
+                                    if (resident === null)
+                                        await mutateTargetConversation(target, () => isolatedConversationAgent(target).clearGoal());
+                                    return { ok: true, cleared: true };
+                                }
+                                if (action === 'flow_pause')
+                                    return await stopFlowForTarget(target);
+                                if (action === 'flow_resume')
+                                    return await resumeFlowForTarget(String(value || ''), target);
+                                if (action === 'conversation_stop') {
+                                    const snapshot = wslBackendEnabled()
+                                        ? await ensureWslConversationPool().snapshot(target)
+                                        : await ensureElectronUtilityPool().snapshot(target);
+                                    const runtime = snapshot.runtime;
+                                    return wslBackendEnabled()
+                                        ? await ensureWslConversationPool().requestStop(target, runtime?.runId)
+                                        : await ensureElectronUtilityPool().requestStop(target, runtime?.runId);
+                                }
+                                if (action === 'input_mode') {
+                                    const mode = String(value || '') === 'next' ? 'next' : 'guide';
+                                    const selected = wslBackendEnabled()
+                                        ? await ensureWslConversationPool().setInputMode(target, mode)
+                                        : await ensureElectronUtilityPool().setInputMode(target, mode);
+                                    return { ok: true, inputMode: selected || mode };
+                                }
+                                const flow = activeFlowStateFor(target);
+                                if (!flow?.abortController || !flow.flowAgent)
+                                    return { ok: false, error: 'No active Flow accepts Guide input.' };
+                                const accepted = flow.flowAgent.queueActiveKernelMessage(String(value || '').trim(), 'steer') || false;
+                                return accepted ? { ok: true, accepted: true, flow: flow.name } : { ok: false, error: 'The current Flow Build is not accepting Guide input.' };
+                            },
+                            conversationPrompt: async (requested, message, requestedOptions) => {
+                                const target = conversationRuntimeTarget(requested);
+                                if (activeFlowStateFor(target))
+                                    clearFlowSuspensionForNewWork(target);
+                                const snapshot = wslBackendEnabled()
+                                    ? await ensureWslConversationPool().snapshot(target)
+                                    : await ensureElectronUtilityPool().snapshot(target);
+                                const requestedMode = String(requestedOptions?.requestedMode || '').toLowerCase();
+                                const goalObjective = String(requestedOptions?.goalObjective || '').trim();
+                                const mode = (goalObjective || requestedMode === 'goal') ? 'build' : String(snapshot.mode || 'build');
+                                const inputMode = String(requestedOptions?.inputMode || snapshot.inputMode || 'guide') === 'next' ? 'next' : 'guide';
+                                const options = {
+                                    mode,
+                                    model: String(snapshot.model || agent.ensureUsableModelSelection()),
+                                    intelligence: String(snapshot.intelligence || agent.intelligence),
+                                    inputMode,
+                                    engine: agent.engine,
+                                };
+                                const queueMode = inputMode === 'guide' ? 'steer' : 'followUp';
+                                const promptMessage = goalObjective ? {
+                                    text: message,
+                                    visibleMode: 'goal',
+                                    goalObjective,
+                                } : message;
+                                const result = wslBackendEnabled()
+                                    ? await ensureWslConversationPool().prompt({
+                                        message: promptMessage,
+                                        target,
+                                        conversationId: target.conversationId,
+                                        options,
+                                        queueMode,
+                                        workspace: target.workspace ? {
+                                            id: target.workspace.id,
+                                            name: target.workspace.name,
+                                            path: target.workspace.path,
+                                            isInternal: target.workspace.isInternal,
+                                            kind: target.workspace.kind,
+                                        } : null,
+                                    })
+                                    : await ensureElectronUtilityPool().prompt({ message: promptMessage, target, options, queueMode });
+                                return { ...result };
+                            },
+                        });
+                        if (agent.config.getBool('remote', 'touch_enabled')) {
                             runServer(root);
                             recordStartup('mobile-server-hosted');
                         }
-                        catch (error) {
-                            console.error('[Newmark] hosted mobile server failed:', error instanceof Error ? error.message : String(error));
-                        }
+                    }
+                    catch (error) {
+                        console.error('[Newmark] hosted mobile server failed:', error instanceof Error ? error.message : String(error));
                     }
                 }
             };
@@ -1976,6 +2164,8 @@ else {
                         return result.tokens.map(token => token.text).join('');
                     });
                     startupAgent.setAutomationManager(automation);
+                    const { setHostedServerAutomation } = require('./server');
+                    setHostedServerAutomation(automation);
                     automation.onChange(items => {
                         setTimeout(() => {
                             try {
@@ -2676,7 +2866,7 @@ else {
                     return await utilityHostToolHandler({ ...request, target: routedTarget, context: routedContext }, signal);
                 }
                 const result = await utilityHostToolHandler({ ...request, target: routedTarget, context: routedContext }, signal);
-                return request.tool === 'computer_use' ? prepareWslComputerUseVisionResult(result) : result;
+                return request.tool === 'computer_use' || request.tool === 'screen_capture' ? prepareWslComputerUseVisionResult(result) : result;
             };
             runWslHostTool.cancelTarget = (runtimeKey) => utilityHostToolHandler.cancelTarget(runtimeKey);
             const ensureWslConversationPool = () => {
@@ -3036,7 +3226,7 @@ else {
                     }
                 }
             });
-            electron_1.ipcMain.handle('flow:resume', async (_event, response, targetInput) => {
+            const resumeFlowForTarget = async (response, targetInput) => {
                 if (!agent)
                     return { ok: false, error: 'No suspended Flow is waiting for user input.' };
                 const requestedTarget = targetInput ? conversationRuntimeTarget(targetInput) : null;
@@ -3170,7 +3360,8 @@ else {
                         await setTargetFlowMode(flowTarget, suspension.previousMode);
                     }
                 }
-            });
+            };
+            electron_1.ipcMain.handle('flow:resume', async (_event, response, targetInput) => await resumeFlowForTarget(response, targetInput));
             electron_1.ipcMain.handle('agent:setMode', async (_event, mode) => {
                 if (agent) {
                     const nextMode = mode;
@@ -3693,10 +3884,31 @@ else {
                         electronUtilityRuntimePool?.updateSetting(section, key, value),
                         wslAgentRuntimePool?.updateSetting(section, key, value),
                     ]);
+                    if (section === 'remote' && key === 'touch_enabled') {
+                        const { setHostedServerEnabled } = require('./server');
+                        await setHostedServerEnabled(value === true || value === 'on');
+                    }
                     return true;
                 }
                 return false;
             });
+            electron_1.ipcMain.handle('mobile:serverStatus', async () => {
+                if (!agent)
+                    return { enabled: false, listening: false, reachable: false, state: 'off', host: '', port: 47890, error: 'Agent not initialized', checkedAt: new Date().toISOString(), startedAt: 0 };
+                const enabled = agent.config.getBool('remote', 'touch_enabled');
+                const { hostedServerStatus } = require('./server');
+                return await hostedServerStatus(enabled);
+            });
+            electron_1.ipcMain.handle('mobile:setRemoteTouchEnabled', async (_event, value) => {
+                if (!agent)
+                    return { ok: false, error: 'Agent not initialized' };
+                const enabled = value === true;
+                agent.config.set('remote', 'touch_enabled', enabled);
+                agent.config.save();
+                const { setHostedServerEnabled } = require('./server');
+                const status = await setHostedServerEnabled(enabled);
+                return { ok: true, ...status };
+            });
             electron_1.ipcMain.handle('agent:openGlobalConfig', async () => {
                 if (!agent)
                     return { error: 'Agent is not initialized' };
@@ -3743,7 +3955,7 @@ else {
                 const accepted = state.flowAgent.queueActiveKernelMessage(text, 'steer') || false;
                 return accepted ? { ok: true, accepted: true, flow: state.name } : { ok: false, error: 'The current Flow Build is not accepting Guide input.' };
             });
-            electron_1.ipcMain.handle('flow:stop', async (_event, targetInput) => {
+            const stopFlowForTarget = async (targetInput) => {
                 if (!agent)
                     return { ok: true, action: 'not_running' };
                 const target = targetInput
@@ -3781,7 +3993,8 @@ else {
                 controller.abort(new Error(`Flow interrupted by user: ${flowName}`));
                 state.flowAgent?.abortActiveKernelRun();
                 return { ok: true, action: 'stopping', flow: flowName };
-            });
+            };
+            electron_1.ipcMain.handle('flow:stop', async (_event, targetInput) => await stopFlowForTarget(targetInput));
             electron_1.ipcMain.handle('agent:readGlobalPrompt', async () => {
                 if (!agent)
                     return { error: 'Agent is not initialized' };

package/dist/preload.js

@@ -139,6 +139,8 @@ contextBridge.exposeInMainWorld('api', {
     updateVersion: () => ipcRenderer.invoke('update:version'),
     mobilePairingQr: () => ipcRenderer.invoke('mobile:pairingQr'),
     mobilePairingStatus: () => ipcRenderer.invoke('mobile:pairingStatus'),
+    mobileServerStatus: () => ipcRenderer.invoke('mobile:serverStatus'),
+    setRemoteTouchEnabled: (enabled) => ipcRenderer.invoke('mobile:setRemoteTouchEnabled', enabled),
     updateCheckGithub: (input) => ipcRenderer.invoke('update:checkGithub', input),
     updateApplyGithub: (input) => ipcRenderer.invoke('update:applyGithub', input),
     updateInstallLocal: (input) => ipcRenderer.invoke('update:installLocal', input),

package/dist/server.d.ts

@@ -1,3 +1,49 @@
+import { Agent, AgentWorkEvent } from './core/agent';
+import { AutomationManager } from './core/automation';
+export interface HostedMobileServerOptions {
+    agent?: Agent;
+    automation?: AutomationManager | null;
+    /** Mobile-originated runs must reach the already-open desktop renderer. */
+    onWorkEvent?: (event: AgentWorkEvent) => void;
+    /** Desktop GUI/runtime-pool events must reach mobile SSE subscribers. */
+    subscribeWorkEvents?: (listener: (event: AgentWorkEvent) => void) => (() => void);
+    /** Read the GUI runtime-pool state for one exact workspace/conversation. */
+    conversationUiState?: (target: {
+        workspaceId: string;
+        conversationId: string;
+    }) => Promise<Record<string, unknown>>;
+    /** Mutate Goal/Flow state owned by the GUI runtime pool for one exact target. */
+    conversationUiAction?: (target: {
+        workspaceId: string;
+        conversationId: string;
+    }, action: 'goal_update' | 'goal_guide' | 'conversation_guide' | 'goal_toggle_pause' | 'goal_clear' | 'flow_pause' | 'flow_resume' | 'flow_guide' | 'conversation_stop' | 'input_mode' | 'queue_enqueue' | 'queue_update' | 'queue_delete' | 'queue_reorder' | 'queue_toggle_pause' | 'queue_guide', value?: string, input?: Record<string, unknown>) => Promise<Record<string, unknown>>;
+    /** Send through the same GUI runtime pool used by the desktop renderer. */
+    conversationPrompt?: (target: {
+        workspaceId: string;
+        conversationId: string;
+    }, message: string, options?: {
+        requestedMode?: string;
+        goalObjective?: string;
+        inputMode?: string;
+    }) => Promise<Record<string, unknown>>;
+}
+export interface HostedMobileServerStatus {
+    enabled: boolean;
+    listening: boolean;
+    reachable: boolean;
+    state: 'off' | 'listening' | 'error';
+    host: string;
+    port: number;
+    error: string;
+    checkedAt: string;
+    startedAt: number;
+}
+export declare function configureHostedServer(root: string, options?: HostedMobileServerOptions): void;
+export declare function stopHostedServer(): Promise<void>;
+export declare function hostedServerStatus(enabled?: boolean, probeHost?: string): Promise<HostedMobileServerStatus>;
+export declare function setHostedServerEnabled(enabled: boolean, probeHost?: string): Promise<HostedMobileServerStatus>;
 /** 托管启动 server（GUI/TUI 内嵌调用；幂等防重入，进程常驻即服务常驻） */
-export declare function runServer(root: string): void;
+export declare function runServer(root: string, options?: HostedMobileServerOptions): void;
+/** Attach the GUI-owned automation manager after deferred startup. */
+export declare function setHostedServerAutomation(manager: AutomationManager): void;
 //# sourceMappingURL=server.d.ts.map