newmark-agent 0.4.7 → 0.4.8

package/dist/server.js

@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runServer = runServer;
+exports.setHostedServerAutomation = setHostedServerAutomation;
 const http = __importStar(require("http"));
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
@@ -56,6 +57,11 @@ let mobileToken = '';
 let appRoot = '';
 const mobileWorkEventSubscribers = new Set();
 const mobileScopedRuntimes = new Map();
+let hostedWorkEventSink = null;
+let unsubscribeHostedWorkEvents = null;
+let hostedConversationUiState;
+let hostedConversationUiAction;
+let hostedConversationPrompt;
 function mobileRuntimeKey(workspaceId, conversationId) {
     return `${String(workspaceId || '')}::${String(conversationId || '')}`;
 }
@@ -67,6 +73,10 @@ function publishMobileWorkEvent(event) {
         catch { /* ignore disconnected mobile listeners */ }
     }
 }
+function publishServerWorkEvent(event) {
+    publishMobileWorkEvent(event);
+    hostedWorkEventSink?.(event);
+}
 function mobileAuthorized(req) {
     if (!mobileToken)
         return false;
@@ -837,6 +847,9 @@ async function handleApi(req, res, body) {
                     mode: agent.mode,
                     model: agent.modelSelectionValue(),
                     modelLabel: agent.modelLabel(),
+                    models: agent.allModelNames(),
+                    providers: (0, config_1.sanitizeProvidersForState)(agent.config.providers()),
+                    intelligence: agent.intelligence,
                     status: agent.status,
                     activeConversationId: agent.activeConversationId,
                     conversations: agent.listConversationStates(),
@@ -876,9 +889,18 @@ async function handleApi(req, res, body) {
                     mobileJson(res, { error: 'Conversation not found in workspace' }, 404);
                     return;
                 }
-                const snapshot = current.getConversationSnapshot(String(conversationId), { window: windowSize, before, workspace: ws });
+                // GUI conversations normally execute inside isolated runtime pools.
+                // Their durable target state can be newer than the hosted main Agent's
+                // foreground memory, so workspace-bound mobile reads must hydrate a
+                // fresh read-only target Agent from storage.
+                const snapshotAgent = ws ? mobileScopedAgent(ws, String(conversationId)) : current;
+                const snapshot = snapshotAgent.getConversationSnapshot(String(conversationId), {
+                    window: windowSize,
+                    before,
+                    workspace: ws || undefined,
+                });
                 // 完整对话信息：workRuns 统一用持久化记录透出（含被中断的构建）
-                const persistedRuns = current.getPersistedConversationWorkRuns(String(conversationId), ws || undefined);
+                const persistedRuns = snapshotAgent.getPersistedConversationWorkRuns(String(conversationId), ws || undefined);
                 if (persistedRuns.length)
                     snapshot.workRuns = persistedRuns;
                 mobileJson(res, snapshot);
@@ -915,6 +937,36 @@ async function handleApi(req, res, body) {
                 mobileJson(res, mobileRightSidebarState(agent, ws, conversationId));
                 return;
             }
+            case '/api/mobile/conversation-ui-state': {
+                const workspaceId = url.searchParams.get('workspaceId') || '';
+                const conversationId = url.searchParams.get('conversationId') || '';
+                const ws = resolveMobileWorkspace(agent, workspaceId);
+                if (!ws) {
+                    mobileJson(res, { error: 'Unknown workspace' }, 404);
+                    return;
+                }
+                if (!conversationId || !agent.hasConversationInWorkspace(conversationId, ws)) {
+                    mobileJson(res, { error: 'Conversation not found in workspace' }, 404);
+                    return;
+                }
+                if (hostedConversationUiState) {
+                    mobileJson(res, await hostedConversationUiState({ workspaceId, conversationId }));
+                    return;
+                }
+                const scoped = mobileScopedAgent(ws, conversationId);
+                const snapshot = scoped.getConversationSnapshot(conversationId, { workspace: ws, window: 1 });
+                mobileJson(res, {
+                    goal: snapshot.goal,
+                    flowSelection: snapshot.flowSelection,
+                    flow: null,
+                    queued: { steering: [], followUp: [] },
+                    runtime: null,
+                    mode: scoped.mode,
+                    status: scoped.status,
+                    inputMode: scoped.inputMode,
+                });
+                return;
+            }
             case '/api/mobile/workspace-files': {
                 const workspaceId = url.searchParams.get('workspaceId') || '';
                 const relativePath = url.searchParams.get('path') || '';
@@ -1030,6 +1082,52 @@ async function handleApi(req, res, body) {
                 });
                 return;
             }
+            case '/api/mobile/model': {
+                const next = String(JSON.parse(body || '{}').model || '');
+                if (!next) {
+                    mobileJson(res, { error: 'Model is required' }, 400);
+                    return;
+                }
+                agent.setModel(next);
+                mobileJson(res, {
+                    ok: true,
+                    model: agent.modelSelectionValue(),
+                    modelLabel: agent.modelLabel(),
+                });
+                return;
+            }
+            case '/api/mobile/intelligence': {
+                const tier = String(JSON.parse(body || '{}').tier || 'medium');
+                agent.setIntelligence(tier, true);
+                mobileJson(res, { ok: true, intelligence: agent.intelligence });
+                return;
+            }
+            case '/api/mobile/conversation-ui-action': {
+                const params = JSON.parse(body || '{}');
+                const workspaceId = String(params.workspaceId || '');
+                const conversationId = String(params.conversationId || '');
+                const action = String(params.action || '');
+                const allowed = new Set(['goal_update', 'goal_guide', 'conversation_guide', 'goal_toggle_pause', 'goal_clear', 'flow_pause', 'flow_resume', 'flow_guide', 'conversation_stop', 'input_mode', 'queue_enqueue', 'queue_update', 'queue_delete', 'queue_toggle_pause', 'queue_guide']);
+                if (!workspaceId || !conversationId || !allowed.has(action)) {
+                    mobileJson(res, { error: 'workspaceId, conversationId, and a valid action are required' }, 400);
+                    return;
+                }
+                const ws = resolveMobileWorkspace(agent, workspaceId);
+                if (!ws) {
+                    mobileJson(res, { error: 'Unknown workspace' }, 404);
+                    return;
+                }
+                if (!agent.hasConversationInWorkspace(conversationId, ws)) {
+                    mobileJson(res, { error: 'Conversation not found in workspace' }, 404);
+                    return;
+                }
+                if (!hostedConversationUiAction) {
+                    mobileJson(res, { error: 'This action requires the GUI-hosted mobile server' }, 409);
+                    return;
+                }
+                mobileJson(res, await hostedConversationUiAction({ workspaceId, conversationId }, action, String(params.value || ''), params));
+                return;
+            }
             case '/api/mobile/conversation-rename': {
                 const params = JSON.parse(body || '{}');
                 const workspaceId = String(params.workspaceId || '');
@@ -1144,6 +1242,15 @@ async function handleApi(req, res, body) {
                         requestAgent = mobileScopedAgent(ws, conversationId);
                     }
                 }
+                if (hostedConversationPrompt && resolvedWorkspace) {
+                    const result = await hostedConversationPrompt({ workspaceId, conversationId }, message, {
+                        requestedMode: String(params.requestedMode || ''),
+                        goalObjective: String(params.goalObjective || ''),
+                        inputMode: String(params.inputMode || ''),
+                    });
+                    mobileJson(res, { ok: true, conversationId, ...result });
+                    return;
+                }
                 if (requestAgent === agent && params.conversationId)
                     requestAgent.setConversation(conversationId);
                 let scopedRuntimeKey = '';
@@ -1151,7 +1258,7 @@ async function handleApi(req, res, body) {
                 let unsubscribeScoped = null;
                 if (requestAgent !== agent && requestWorkspace) {
                     scopedRuntimeKey = mobileRuntimeKey(String(requestWorkspace.id || ''), conversationId);
-                    unsubscribeScoped = requestAgent.subscribeWorkEvents(event => publishMobileWorkEvent({
+                    unsubscribeScoped = requestAgent.subscribeWorkEvents(event => publishServerWorkEvent({
                         ...event,
                         workspaceId: String(requestWorkspace?.id || ''),
                         conversationId,
@@ -1302,40 +1409,51 @@ async function handleApi(req, res, body) {
             jsonResponse(res, { error: e.message }, 500);
     }
 }
-function startServer(root) {
+function startServer(root, options = {}) {
     mobileToken = (0, mobilePairing_1.ensureMobileToken)(root);
     appRoot = root;
-    agent = new agent_1.Agent(root);
+    agent = options.agent || new agent_1.Agent(root);
+    hostedWorkEventSink = options.onWorkEvent || null;
+    hostedConversationUiState = options.conversationUiState;
+    hostedConversationUiAction = options.conversationUiAction;
+    hostedConversationPrompt = options.conversationPrompt;
+    unsubscribeHostedWorkEvents?.();
+    unsubscribeHostedWorkEvents = options.subscribeWorkEvents?.(publishMobileWorkEvent) || null;
     workspaceFileRouter = new workspaceFileRouter_1.WorkspaceFileRouter(() => path.resolve(agent?.workspace.current?.path || root));
-    automation = new automation_1.AutomationManager(agent.config, async (prompt, model, item) => {
-        if (!agent)
-            return '';
-        const previousModel = agent.model;
-        const previousWorkspace = agent.workspace.current?.id || agent.workspace.current?.path || '';
-        const previousConversation = agent.activeConversationId;
-        agent.selectWorkspaceFromStorage(item.workspaceId);
-        const conversationId = item.conversationMode === 'existing'
-            ? item.conversationId
-            : `automation-${item.id}-${Date.now().toString(36)}`;
-        agent.setConversationFromStorage(conversationId);
-        automation?.update(item.id, { lastConversationId: conversationId });
-        if (model)
-            agent.setModel(model);
-        try {
-            const tokens = await agent.process(prompt);
-            return tokens.map(t => t.text).join('');
-        }
-        finally {
+    automation = options.automation || null;
+    if (!automation && !options.agent)
+        automation = new automation_1.AutomationManager(agent.config, async (prompt, model, item) => {
+            if (!agent)
+                return '';
+            const previousModel = agent.model;
+            const previousWorkspace = agent.workspace.current?.id || agent.workspace.current?.path || '';
+            const previousConversation = agent.activeConversationId;
+            agent.selectWorkspaceFromStorage(item.workspaceId);
+            const conversationId = item.conversationMode === 'existing'
+                ? item.conversationId
+                : `automation-${item.id}-${Date.now().toString(36)}`;
+            agent.setConversationFromStorage(conversationId);
+            automation?.update(item.id, { lastConversationId: conversationId });
             if (model)
-                agent.setModel(previousModel);
-            if (previousWorkspace)
-                agent.selectWorkspaceFromStorage(previousWorkspace);
-            agent.setConversationFromStorage(previousConversation);
-        }
-    });
-    agent.subscribeWorkEvents(publishMobileWorkEvent);
-    agent.setAutomationManager(automation);
-    automation.start();
+                agent.setModel(model);
+            try {
+                const tokens = await agent.process(prompt);
+                return tokens.map(t => t.text).join('');
+            }
+            finally {
+                if (model)
+                    agent.setModel(previousModel);
+                if (previousWorkspace)
+                    agent.selectWorkspaceFromStorage(previousWorkspace);
+                agent.setConversationFromStorage(previousConversation);
+            }
+        });
+    agent.subscribeWorkEvents(publishServerWorkEvent);
+    if (automation) {
+        agent.setAutomationManager(automation);
+        if (!options.automation)
+            automation.start();
+    }
     const uiDir = path.join(__dirname, 'ui');
     const server = http.createServer(async (req, res) => {
         if (req.method === 'OPTIONS') {
@@ -1388,10 +1506,14 @@ function startServer(root) {
 }
 let hostedServerStarted = false;
 /** 托管启动 server（GUI/TUI 内嵌调用；幂等防重入，进程常驻即服务常驻） */
-function runServer(root) {
+function runServer(root, options = {}) {
     if (hostedServerStarted)
         return;
     hostedServerStarted = true;
-    startServer(root);
+    startServer(root, options);
+}
+/** Attach the GUI-owned automation manager after deferred startup. */
+function setHostedServerAutomation(manager) {
+    automation = manager;
 }
 //# sourceMappingURL=server.js.map

package/dist/tools/index.js

@@ -294,6 +294,13 @@ class ToolExecutor {
                 max_refs: { type: 'number' },
                 attribute: { type: 'string' },
             }, ['action']),
+            t('screen_capture', 'Read-only active Windows screenshot available without provisioning or starting full Computer Use. Capture the whole desktop or one visible application. The image is a one-use model input and is deleted immediately; semantic Windows UI objects are returned alongside it.', {
+                target: { type: 'string', enum: ['desktop', 'application'], description: 'Capture the whole desktop or one visible application. Defaults to desktop.' },
+                app_target: { type: 'string', description: 'For target=application: application title, process name, process id, or window handle.' },
+                capture_max_width: { type: 'number', minimum: 320, maximum: 2048, description: 'Maximum ephemeral screenshot width; defaults to 1280. Aspect ratio is preserved and the source is never enlarged.' },
+                capture_max_height: { type: 'number', minimum: 240, maximum: 2048, description: 'Maximum ephemeral screenshot height; defaults to 960. Aspect ratio is preserved and the source is never enlarged.' },
+                max_chars: { type: 'number', minimum: 1000, maximum: 100000, description: 'Maximum returned semantic UI text characters.' },
+            }, []),
             t('computer_use', 'Native Windows desktop control with persistent observation/action helpers. Use observe/app_observe before acting. sequence may perform up to three stable low-risk move/click/scroll/wait/app_activate steps and stops when focus, window, menu, dialog, scene, or risk changes. Vision screenshots remain one-time inputs and are deleted immediately.', {
                 action: { type: 'string', enum: ['observe', 'app_list', 'app_observe', 'sequence', 'takeover_start', 'takeover_stop', 'move', 'click', 'scroll', 'type', 'key', 'wait', 'app_activate', 'app_click', 'app_scroll', 'app_type', 'app_key'] },
                 capture_max_width: { type: 'number', minimum: 320, maximum: 2048, description: 'For observe/app_observe only. Maximum ephemeral screenshot width; defaults to 1280. Aspect ratio is preserved and the source is never enlarged.' },
@@ -790,6 +797,44 @@ class ToolExecutor {
                         result: parsed,
                     }, null, 2);
                 }
+                case 'screen_capture': {
+                    const target = g('target').toLowerCase() === 'application' ? 'application' : 'desktop';
+                    if (process.env.NEWMARK_WSL_DISTRO) {
+                        const result = await (0, wslHostToolBridge_1.requestWindowsHostTool)('screen_capture', args, {
+                            conversationId: context.conversationId || 'default',
+                            workspaceId: context.workspaceId || (0, terminalTakeover_1.terminalTakeoverWorkspaceId)(context.workspacePath || wsPath),
+                            actorId: context.actorId || terminalTakeover_1.ROOT_TERMINAL_ACTOR_ID,
+                            runtimeKey: browserUseScope(context, wsPath).runtimeKey,
+                            allowEphemeralVisionImage: context.allowEphemeralVisionImage === true,
+                            mode: context.mode || 'build',
+                        }, 120_000, context.signal);
+                        return typeof result === 'string' ? result : JSON.stringify(result);
+                    }
+                    if (process.env.NEWMARK_ISOLATED_RUNTIME === '1') {
+                        const result = await (0, utilityHostToolBridge_1.requestUtilityHostTool)('screen_capture', args, {
+                            conversationId: context.conversationId || 'default',
+                            workspaceId: context.workspaceId || (0, terminalTakeover_1.terminalTakeoverWorkspaceId)(context.workspacePath || wsPath),
+                            actorId: context.actorId || terminalTakeover_1.ROOT_TERMINAL_ACTOR_ID,
+                            workspacePath: context.workspacePath || wsPath,
+                            backend: 'utility',
+                            mode: context.mode || 'build',
+                            allowEphemeralVisionImage: context.allowEphemeralVisionImage === true,
+                        }, 120_000, context.signal);
+                        return typeof result === 'string' ? result : JSON.stringify(result);
+                    }
+                    return await (0, computerUse_1.runComputerUse)({
+                        action: target === 'application' ? 'app_observe' : 'observe',
+                        appTarget: g('app_target'),
+                        windowHandle: target === 'application' ? g('app_target') : '',
+                        maxChars: Number(args.max_chars || 30000),
+                        workspacePath: wsPath,
+                        allowEphemeralVisionImage: context.allowEphemeralVisionImage === true,
+                        captureMaxWidth: Number(args.capture_max_width),
+                        captureMaxHeight: Number(args.capture_max_height),
+                        invocation: context.invocation,
+                        ownerId: `screen-capture:${computerUseOwner(context, wsPath)}:${String(context.actorId || 'root')}`,
+                    });
+                }
                 case 'computer_use': {
                     const action = normalizeComputerUseAction(g('action'));
                     const owner = `${computerUseOwner(context, wsPath)}:${String(context.actorId || 'root')}`;
@@ -976,7 +1021,7 @@ class ToolExecutor {
     hostSupportsTool(name) {
         if (name.startsWith('browser_') && !this.hostProfile.electronBrowser)
             return false;
-        if (name === 'computer_use' && !this.hostProfile.windowsComputerUse)
+        if ((name === 'computer_use' || name === 'screen_capture') && !this.hostProfile.windowsComputerUse)
             return false;
         return true;
     }

package/dist/tools/nativeTools.js

@@ -26,6 +26,7 @@ exports.NATIVE_TOOL_CATALOG = [
     { name: 'browser_reload', label: 'Browser reload', description: 'Reload the built-in browser.', category: 'browser', defaultEnabled: true },
     { name: 'browser_cdp', label: 'Browser CDP', description: 'Run an advanced Chrome DevTools Protocol command.', category: 'browser', defaultEnabled: true },
     { name: 'browser_use', label: 'Browser Use', description: 'Observe and control the built-in browser through scoped opaque element refs and structured receipts.', category: 'browser', defaultEnabled: true },
+    { name: 'screen_capture', label: 'Active screenshot', description: 'Capture the Windows desktop or one visible application without starting the full Computer Use control surface.', category: 'desktop', defaultEnabled: true, protected: true, availability: 'mode-scoped' },
     { name: 'computer_use', label: 'Computer Use', description: 'Observe and control Windows desktop UI with screenshots and semantic objects.', category: 'desktop', defaultEnabled: true },
     { name: 'image_inspect', label: 'Image inspect', description: 'Crop and magnify images submitted in the current conversation for closer visual inspection.', category: 'core', defaultEnabled: true, protected: true, availability: 'mode-scoped' },
     { name: 'image_display', label: 'Image display', description: 'Present a workspace PNG/JPEG inside the current Build Block.', category: 'core', defaultEnabled: true, protected: true, availability: 'mode-scoped' },