newmark-agent 0.4.7 → 0.4.8

package/dist/core/utilityHostToolRouter.js

@@ -126,6 +126,24 @@ function createUtilityHostToolHandler(options) {
             throwIfAborted(signal);
             return result;
         }
+        if (request.tool === 'screen_capture') {
+            const args = request.args || {};
+            const target = String(args.target || '').toLowerCase() === 'application' ? 'application' : 'desktop';
+            const result = await (options.runComputer || computerUse_1.runComputerUse)({
+                action: target === 'application' ? 'app_observe' : 'observe',
+                appTarget: String(args.app_target || args.appTarget || ''),
+                windowHandle: target === 'application' ? String(args.app_target || args.appTarget || '') : '',
+                maxChars: Number(args.max_chars || args.maxChars || 30_000),
+                captureMaxWidth: Number(args.capture_max_width || args.captureMaxWidth),
+                captureMaxHeight: Number(args.capture_max_height || args.captureMaxHeight),
+                allowEphemeralVisionImage: request.context.allowEphemeralVisionImage === true,
+                workspacePath: request.target.workspacePath,
+                invocation: 'agent',
+                ownerId: `screen-capture:${request.target.runtimeKey}:${String(request.context.actorId || ROOT_AGENT_ACTOR_ID)}`,
+            });
+            throwIfAborted(signal);
+            return result;
+        }
         const args = request.args || {};
         const action = String(args.action || '').trim().toLowerCase();
         const trustedComputerUseContext = request.context;

package/dist/core/wslAgentClient.d.ts

@@ -1,4 +1,5 @@
 import { AgentMode, AgentWorkEvent, ConversationInputEnvelope, GuideReceipt } from './types';
+import { ConversationQueueAction } from './conversationKernel';
 import { TerminalTakeoverEvent, TerminalTakeoverOwnerFilter, TerminalTakeoverState } from '../tools/terminalTakeover';
 import { WslAgentPromptRequest, WslAgentPromptResult, WslAutoRouteRatingResult, WslAgentWorkspace, WslAgentStopResult, WslConversationRewindResult, WslHostToolRequest } from './wslAgentProtocol';
 import { ConversationRuntimeTarget, NormalizedConversationTarget } from './conversationTarget';
@@ -71,6 +72,13 @@ export declare class WslAgentClient {
     snapshotTarget(target: ConversationRuntimeTarget): Promise<Record<string, unknown>>;
     rewind(target: ConversationRuntimeTarget, messageIndex: number): Promise<WslConversationRewindResult>;
     enqueueGuide(target: ConversationRuntimeTarget, envelope: ConversationInputEnvelope): Promise<GuideReceipt>;
+    queueAction(target: ConversationRuntimeTarget, action: ConversationQueueAction, input?: {
+        id?: string;
+        text?: string;
+        requestedMode?: string;
+        goalObjective?: string;
+        createdAt?: string;
+    }): Promise<Record<string, unknown>>;
     checkpoint(target: ConversationRuntimeTarget): Promise<Record<string, unknown>>;
     contextCompress(target: ConversationRuntimeTarget, options?: {
         keepRecent?: number;

package/dist/core/wslAgentClient.js

@@ -313,6 +313,10 @@ class WslAgentClient {
         await this.start();
         return await this.request('guide', { target: await this.mapTarget(target), envelope }, 5_000);
     }
+    async queueAction(target, action, input = {}) {
+        await this.start();
+        return await this.request('queue_action', { target: await this.mapTarget(target), action, input }, 5_000);
+    }
     async checkpoint(target) {
         await this.start();
         return await this.request('checkpoint', { target: await this.mapTarget(target) }, 5_000);
@@ -536,7 +540,7 @@ class WslAgentClient {
         else if (!this.hostToolHandler) {
             result = { requestId: request.requestId, ok: false, error: 'No Windows host tool handler is registered' };
         }
-        else if (!['browser_control', 'computer_use', 'browser_use', 'automation', 'terminal_takeover'].includes(request.tool)) {
+        else if (!['browser_control', 'screen_capture', 'computer_use', 'browser_use', 'automation', 'terminal_takeover'].includes(request.tool)) {
             result = { requestId: request.requestId, ok: false, error: `WSL host tool is not allowed: ${String(request.tool)}` };
         }
         else {

package/dist/core/wslAgentProtocol.d.ts

@@ -1,5 +1,5 @@
 import { AgentMode, AgentWorkEvent, ConversationInputEnvelope, GuideReceipt } from './types';
-import { AgentPromptMessage, ConversationContextCompressOptions, ConversationKernelRunOptions, ConversationKernelRunResult, ConversationQueueMode, ConversationStopResult } from './conversationKernel';
+import { AgentPromptMessage, ConversationContextCompressOptions, ConversationKernelRunOptions, ConversationKernelRunResult, ConversationQueueAction, ConversationQueueMode, ConversationStopResult } from './conversationKernel';
 import { ConversationRuntimeTarget } from './conversationTarget';
 import { TerminalTakeoverEvent, TerminalTakeoverOwnerFilter, TerminalTakeoverState } from '../tools/terminalTakeover';
 import { BrowserUseRequest } from './browserUse';
@@ -42,6 +42,9 @@ interface WslHostToolRequestBase {
 export type WslHostToolRequest = (WslHostToolRequestBase & {
     tool: 'browser_control';
     args: BrowserControlRequest;
+}) | (WslHostToolRequestBase & {
+    tool: 'screen_capture';
+    args: Record<string, unknown>;
 }) | (WslHostToolRequestBase & {
     tool: 'computer_use';
     args: Record<string, unknown>;
@@ -108,6 +111,20 @@ export type WslAgentRequest = {
         target: ConversationRuntimeTarget;
         envelope: ConversationInputEnvelope;
     };
+} | {
+    id: string;
+    method: 'queue_action';
+    params: {
+        target: ConversationRuntimeTarget;
+        action: ConversationQueueAction;
+        input?: {
+            id?: string;
+            text?: string;
+            requestedMode?: string;
+            goalObjective?: string;
+            createdAt?: string;
+        };
+    };
 } | {
     id: string;
     method: 'checkpoint';

package/dist/core/wslAgentRuntimePool.d.ts

@@ -1,4 +1,5 @@
 import { AgentMode, AgentWorkEvent, ConversationInputEnvelope, GuideReceipt } from './types';
+import { ConversationQueueAction } from './conversationKernel';
 import { ConversationRuntimeTarget, NormalizedConversationTarget } from './conversationTarget';
 import { WslHostToolHandler } from './wslAgentClient';
 import { WslAgentPromptRequest, WslAgentPromptResult, WslAutoRouteRatingResult, WslAgentStopResult, WslConversationRewindResult } from './wslAgentProtocol';
@@ -10,6 +11,13 @@ export interface WslTargetRuntimeClient {
     rewind(target: ConversationRuntimeTarget, messageIndex: number): Promise<WslConversationRewindResult>;
     requestStop(target: ConversationRuntimeTarget, runId?: string): Promise<WslAgentStopResult>;
     enqueueGuide(target: ConversationRuntimeTarget, envelope: ConversationInputEnvelope): Promise<GuideReceipt>;
+    queueAction?(target: ConversationRuntimeTarget, action: ConversationQueueAction, input?: {
+        id?: string;
+        text?: string;
+        requestedMode?: string;
+        goalObjective?: string;
+        createdAt?: string;
+    }): Promise<Record<string, unknown>>;
     checkpoint(target: ConversationRuntimeTarget): Promise<Record<string, unknown>>;
     contextCompress?(target: ConversationRuntimeTarget, options?: {
         keepRecent?: number;
@@ -73,6 +81,13 @@ export declare class WslAgentRuntimePool {
     rewind(target: ConversationRuntimeTarget, messageIndex: number): Promise<WslConversationRewindResult>;
     requestStop(target: ConversationRuntimeTarget, runId?: string): Promise<WslPoolStopResult>;
     enqueueGuide(envelope: ConversationInputEnvelope): Promise<GuideReceipt>;
+    queueAction(target: ConversationRuntimeTarget, action: ConversationQueueAction, input?: {
+        id?: string;
+        text?: string;
+        requestedMode?: string;
+        goalObjective?: string;
+        createdAt?: string;
+    }): Promise<Record<string, unknown>>;
     checkpoint(target: ConversationRuntimeTarget): Promise<Record<string, unknown>>;
     contextCompress(target: ConversationRuntimeTarget, options?: {
         keepRecent?: number;

package/dist/core/wslAgentRuntimePool.js

@@ -195,6 +195,17 @@ class WslAgentRuntimePool {
             this.release(entry, true);
         }
     }
+    async queueAction(target, action, input = {}) {
+        const entry = await this.acquireExisting(target);
+        if (!entry || !entry.client.queueAction)
+            throw new Error('Target conversation is not running');
+        try {
+            return await entry.client.queueAction(entry.target, action, input);
+        }
+        finally {
+            this.release(entry, true);
+        }
+    }
     async checkpoint(target) {
         const normalized = (0, conversationTarget_1.normalizeConversationTarget)(target);
         const entry = await this.acquireExisting(normalized);

package/dist/main.js

@@ -120,6 +120,7 @@ let _forceQuit = false;
 let forcedExitTimer = null;
 let electronBrowserUseHost = null;
 let browserUseEngine = null;
+const mobileServerWorkEventSubscribers = new Set();
 // A single renderer can host several hidden conversation-bound guests. The
 // legacy host map is retained for focused-window discovery, while this map is
 // the authoritative Browser-Use/right-sidebar binding.
@@ -288,7 +289,7 @@ async function resetWslAgentClient() {
     if (wslAgentClient)
         await wslAgentClient.resetAgent();
 }
-function broadcastAgentWorkEvent(event) {
+function broadcastAgentWorkEvent(event, mirrorToMobile = true) {
     const workEvent = event;
     if ((workEvent.type === 'done' || workEvent.type === 'error') && workEvent.runtimeKey) {
         browserUseEngine?.clearRuntime(workEvent.runtimeKey);
@@ -299,6 +300,14 @@ function broadcastAgentWorkEvent(event) {
             continue;
         win.webContents.send('agent:workEvent', event);
     }
+    if (mirrorToMobile) {
+        for (const listener of mobileServerWorkEventSubscribers) {
+            try {
+                listener(event);
+            }
+            catch { /* ignore disconnected mobile bridge */ }
+        }
+    }
 }
 function ensureConversationKernel(root) {
     if (!agent)
@@ -1886,7 +1895,180 @@ else {
                     if (agent.config.getBool('remote', 'touch_enabled')) {
                         try {
                             const { runServer } = require('./server');
-                            runServer(root);
+                            runServer(root, {
+                                agent,
+                                automation,
+                                onWorkEvent: event => broadcastAgentWorkEvent(event, false),
+                                subscribeWorkEvents: listener => {
+                                    mobileServerWorkEventSubscribers.add(listener);
+                                    return () => mobileServerWorkEventSubscribers.delete(listener);
+                                },
+                                conversationUiState: async (requested) => {
+                                    const target = conversationRuntimeTarget(requested);
+                                    const snapshot = wslBackendEnabled()
+                                        ? await ensureWslConversationPool().snapshot(target)
+                                        : await ensureElectronUtilityPool().snapshot(target);
+                                    const flowRunning = flowRunningForTarget(target);
+                                    const flowSuspension = flowSuspensionForTarget(target);
+                                    return {
+                                        ...snapshot,
+                                        flow: flowRunning ? {
+                                            running: true,
+                                            paused: false,
+                                            name: flowRunning.name,
+                                            promptText: String(activeFlowStateFor(target)?.input || ''),
+                                            message: '',
+                                        } : flowSuspension ? {
+                                            running: true,
+                                            paused: true,
+                                            name: flowSuspension.workflowName || '',
+                                            promptText: String(flowSuspension.input || ''),
+                                            message: String(flowSuspension.message || ''),
+                                            reason: flowSuspension.reason || '',
+                                        } : null,
+                                    };
+                                },
+                                conversationUiAction: async (requested, action, value, input) => {
+                                    const target = conversationRuntimeTarget(requested);
+                                    if (action.startsWith('queue_')) {
+                                        const queueAction = action === 'queue_enqueue' ? 'enqueue'
+                                            : action === 'queue_update' ? 'update'
+                                                : action === 'queue_delete' ? 'delete'
+                                                    : action === 'queue_toggle_pause' ? 'toggle_pause'
+                                                        : 'guide';
+                                        const queueInput = {
+                                            id: String(input?.id || ''),
+                                            text: String(input?.text || value || ''),
+                                            requestedMode: String(input?.requestedMode || 'build'),
+                                            goalObjective: String(input?.goalObjective || ''),
+                                            createdAt: String(input?.createdAt || ''),
+                                        };
+                                        return wslBackendEnabled()
+                                            ? await ensureWslConversationPool().queueAction(target, queueAction, queueInput)
+                                            : await ensureElectronUtilityPool().queueAction(target, queueAction, queueInput);
+                                    }
+                                    if (action === 'goal_update') {
+                                        return { goal: await mutateTargetConversation(target, () => {
+                                                const isolated = isolatedConversationAgent(target);
+                                                isolated.updateGoal(String(value || '').trim());
+                                                isolated.setMode('goal');
+                                                isolated.saveWorkspaceConversationState(true);
+                                                return isolated.getConversationSnapshot(target.conversationId).goal;
+                                            }) };
+                                    }
+                                    if (action === 'goal_guide' || action === 'conversation_guide') {
+                                        const objective = action === 'goal_guide' ? String(value || '').trim() : '';
+                                        const guideText = String(value || '').trim();
+                                        const snapshot = wslBackendEnabled()
+                                            ? await ensureWslConversationPool().snapshot(target)
+                                            : await ensureElectronUtilityPool().snapshot(target);
+                                        const runtime = snapshot.runtime;
+                                        if (!guideText || !runtime?.running || !runtime.runId) {
+                                            return { ok: false, error: action === 'goal_guide'
+                                                    ? 'There is no active Build for this Goal Guide.'
+                                                    : 'Target conversation is not running.' };
+                                        }
+                                        const prefix = 'Goal for the current Build:\n';
+                                        const now = new Date().toISOString();
+                                        const envelope = {
+                                            clientMessageId: (0, crypto_1.randomUUID)(),
+                                            guideId: (0, crypto_1.randomUUID)(),
+                                            target: { workspaceId: target.workspaceId, conversationId: target.conversationId },
+                                            runId: runtime.runId,
+                                            deliveryMode: 'steer',
+                                            text: objective ? prefix + objective : guideText,
+                                            goalObjective: objective || undefined,
+                                            createdAt: now,
+                                        };
+                                        const receipt = wslBackendEnabled()
+                                            ? await ensureWslConversationPool().enqueueGuide(envelope)
+                                            : await ensureElectronUtilityPool().enqueueGuide(envelope);
+                                        return { ok: receipt.status !== 'rejected', receipt };
+                                    }
+                                    if (action === 'goal_toggle_pause') {
+                                        const resident = wslBackendEnabled()
+                                            ? await ensureWslConversationPool().toggleGoalPause(target)
+                                            : await ensureElectronUtilityPool().toggleGoalPause(target);
+                                        const paused = resident !== null ? resident : await mutateTargetConversation(target, () => isolatedConversationAgent(target).toggleGoalPause());
+                                        return { ok: true, paused };
+                                    }
+                                    if (action === 'goal_clear') {
+                                        const resident = wslBackendEnabled()
+                                            ? await ensureWslConversationPool().clearGoal(target)
+                                            : await ensureElectronUtilityPool().clearGoal(target);
+                                        if (resident === null)
+                                            await mutateTargetConversation(target, () => isolatedConversationAgent(target).clearGoal());
+                                        return { ok: true, cleared: true };
+                                    }
+                                    if (action === 'flow_pause')
+                                        return await stopFlowForTarget(target);
+                                    if (action === 'flow_resume')
+                                        return await resumeFlowForTarget(String(value || ''), target);
+                                    if (action === 'conversation_stop') {
+                                        const snapshot = wslBackendEnabled()
+                                            ? await ensureWslConversationPool().snapshot(target)
+                                            : await ensureElectronUtilityPool().snapshot(target);
+                                        const runtime = snapshot.runtime;
+                                        return wslBackendEnabled()
+                                            ? await ensureWslConversationPool().requestStop(target, runtime?.runId)
+                                            : await ensureElectronUtilityPool().requestStop(target, runtime?.runId);
+                                    }
+                                    if (action === 'input_mode') {
+                                        const mode = String(value || '') === 'next' ? 'next' : 'guide';
+                                        const selected = wslBackendEnabled()
+                                            ? await ensureWslConversationPool().setInputMode(target, mode)
+                                            : await ensureElectronUtilityPool().setInputMode(target, mode);
+                                        return { ok: true, inputMode: selected || mode };
+                                    }
+                                    const flow = activeFlowStateFor(target);
+                                    if (!flow?.abortController || !flow.flowAgent)
+                                        return { ok: false, error: 'No active Flow accepts Guide input.' };
+                                    const accepted = flow.flowAgent.queueActiveKernelMessage(String(value || '').trim(), 'steer') || false;
+                                    return accepted ? { ok: true, accepted: true, flow: flow.name } : { ok: false, error: 'The current Flow Build is not accepting Guide input.' };
+                                },
+                                conversationPrompt: async (requested, message, requestedOptions) => {
+                                    const target = conversationRuntimeTarget(requested);
+                                    if (activeFlowStateFor(target))
+                                        clearFlowSuspensionForNewWork(target);
+                                    const snapshot = wslBackendEnabled()
+                                        ? await ensureWslConversationPool().snapshot(target)
+                                        : await ensureElectronUtilityPool().snapshot(target);
+                                    const requestedMode = String(requestedOptions?.requestedMode || '').toLowerCase();
+                                    const goalObjective = String(requestedOptions?.goalObjective || '').trim();
+                                    const mode = (goalObjective || requestedMode === 'goal') ? 'build' : String(snapshot.mode || 'build');
+                                    const inputMode = String(requestedOptions?.inputMode || snapshot.inputMode || 'guide') === 'next' ? 'next' : 'guide';
+                                    const options = {
+                                        mode,
+                                        model: String(snapshot.model || agent.ensureUsableModelSelection()),
+                                        intelligence: String(snapshot.intelligence || agent.intelligence),
+                                        inputMode,
+                                        engine: agent.engine,
+                                    };
+                                    const queueMode = inputMode === 'guide' ? 'steer' : 'followUp';
+                                    const promptMessage = goalObjective ? {
+                                        text: message,
+                                        visibleMode: 'goal',
+                                        goalObjective,
+                                    } : message;
+                                    const result = wslBackendEnabled()
+                                        ? await ensureWslConversationPool().prompt({
+                                            message: promptMessage,
+                                            target,
+                                            conversationId: target.conversationId,
+                                            options,
+                                            queueMode,
+                                            workspace: target.workspace ? {
+                                                id: target.workspace.id,
+                                                name: target.workspace.name,
+                                                path: target.workspace.path,
+                                                isInternal: target.workspace.isInternal,
+                                                kind: target.workspace.kind,
+                                            } : null,
+                                        })
+                                        : await ensureElectronUtilityPool().prompt({ message: promptMessage, target, options, queueMode });
+                                    return { ...result };
+                                },
+                            });
                             recordStartup('mobile-server-hosted');
                         }
                         catch (error) {
@@ -1976,6 +2158,8 @@ else {
                         return result.tokens.map(token => token.text).join('');
                     });
                     startupAgent.setAutomationManager(automation);
+                    const { setHostedServerAutomation } = require('./server');
+                    setHostedServerAutomation(automation);
                     automation.onChange(items => {
                         setTimeout(() => {
                             try {
@@ -2676,7 +2860,7 @@ else {
                     return await utilityHostToolHandler({ ...request, target: routedTarget, context: routedContext }, signal);
                 }
                 const result = await utilityHostToolHandler({ ...request, target: routedTarget, context: routedContext }, signal);
-                return request.tool === 'computer_use' ? prepareWslComputerUseVisionResult(result) : result;
+                return request.tool === 'computer_use' || request.tool === 'screen_capture' ? prepareWslComputerUseVisionResult(result) : result;
             };
             runWslHostTool.cancelTarget = (runtimeKey) => utilityHostToolHandler.cancelTarget(runtimeKey);
             const ensureWslConversationPool = () => {
@@ -3036,7 +3220,7 @@ else {
                     }
                 }
             });
-            electron_1.ipcMain.handle('flow:resume', async (_event, response, targetInput) => {
+            const resumeFlowForTarget = async (response, targetInput) => {
                 if (!agent)
                     return { ok: false, error: 'No suspended Flow is waiting for user input.' };
                 const requestedTarget = targetInput ? conversationRuntimeTarget(targetInput) : null;
@@ -3170,7 +3354,8 @@ else {
                         await setTargetFlowMode(flowTarget, suspension.previousMode);
                     }
                 }
-            });
+            };
+            electron_1.ipcMain.handle('flow:resume', async (_event, response, targetInput) => await resumeFlowForTarget(response, targetInput));
             electron_1.ipcMain.handle('agent:setMode', async (_event, mode) => {
                 if (agent) {
                     const nextMode = mode;
@@ -3743,7 +3928,7 @@ else {
                 const accepted = state.flowAgent.queueActiveKernelMessage(text, 'steer') || false;
                 return accepted ? { ok: true, accepted: true, flow: state.name } : { ok: false, error: 'The current Flow Build is not accepting Guide input.' };
             });
-            electron_1.ipcMain.handle('flow:stop', async (_event, targetInput) => {
+            const stopFlowForTarget = async (targetInput) => {
                 if (!agent)
                     return { ok: true, action: 'not_running' };
                 const target = targetInput
@@ -3781,7 +3966,8 @@ else {
                 controller.abort(new Error(`Flow interrupted by user: ${flowName}`));
                 state.flowAgent?.abortActiveKernelRun();
                 return { ok: true, action: 'stopping', flow: flowName };
-            });
+            };
+            electron_1.ipcMain.handle('flow:stop', async (_event, targetInput) => await stopFlowForTarget(targetInput));
             electron_1.ipcMain.handle('agent:readGlobalPrompt', async () => {
                 if (!agent)
                     return { error: 'Agent is not initialized' };

package/dist/server.d.ts

@@ -1,3 +1,34 @@
+import { Agent, AgentWorkEvent } from './core/agent';
+import { AutomationManager } from './core/automation';
+export interface HostedMobileServerOptions {
+    agent?: Agent;
+    automation?: AutomationManager | null;
+    /** Mobile-originated runs must reach the already-open desktop renderer. */
+    onWorkEvent?: (event: AgentWorkEvent) => void;
+    /** Desktop GUI/runtime-pool events must reach mobile SSE subscribers. */
+    subscribeWorkEvents?: (listener: (event: AgentWorkEvent) => void) => (() => void);
+    /** Read the GUI runtime-pool state for one exact workspace/conversation. */
+    conversationUiState?: (target: {
+        workspaceId: string;
+        conversationId: string;
+    }) => Promise<Record<string, unknown>>;
+    /** Mutate Goal/Flow state owned by the GUI runtime pool for one exact target. */
+    conversationUiAction?: (target: {
+        workspaceId: string;
+        conversationId: string;
+    }, action: 'goal_update' | 'goal_guide' | 'conversation_guide' | 'goal_toggle_pause' | 'goal_clear' | 'flow_pause' | 'flow_resume' | 'flow_guide' | 'conversation_stop' | 'input_mode' | 'queue_enqueue' | 'queue_update' | 'queue_delete' | 'queue_toggle_pause' | 'queue_guide', value?: string, input?: Record<string, unknown>) => Promise<Record<string, unknown>>;
+    /** Send through the same GUI runtime pool used by the desktop renderer. */
+    conversationPrompt?: (target: {
+        workspaceId: string;
+        conversationId: string;
+    }, message: string, options?: {
+        requestedMode?: string;
+        goalObjective?: string;
+        inputMode?: string;
+    }) => Promise<Record<string, unknown>>;
+}
 /** 托管启动 server（GUI/TUI 内嵌调用；幂等防重入，进程常驻即服务常驻） */
-export declare function runServer(root: string): void;
+export declare function runServer(root: string, options?: HostedMobileServerOptions): void;
+/** Attach the GUI-owned automation manager after deferred startup. */
+export declare function setHostedServerAutomation(manager: AutomationManager): void;
 //# sourceMappingURL=server.d.ts.map