neoagent 3.2.1-beta.1 → 3.2.1-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (172) hide show
  1. package/extensions/chrome-browser/background.mjs +318 -88
  2. package/extensions/chrome-browser/http.mjs +136 -0
  3. package/extensions/chrome-browser/protocol.mjs +654 -90
  4. package/flutter_app/lib/main_chat.dart +118 -739
  5. package/flutter_app/lib/main_controller.dart +111 -20
  6. package/flutter_app/lib/main_integrations.dart +607 -8
  7. package/flutter_app/lib/main_models.dart +3 -0
  8. package/flutter_app/lib/main_operations.dart +334 -321
  9. package/flutter_app/lib/main_security.dart +266 -112
  10. package/flutter_app/lib/main_settings.dart +4 -3
  11. package/flutter_app/lib/main_shared.dart +14 -11
  12. package/flutter_app/lib/src/backend_client.dart +78 -0
  13. package/flutter_app/lib/src/desktop_companion_actions.dart +185 -31
  14. package/flutter_app/lib/src/desktop_companion_io.dart +319 -86
  15. package/flutter_app/windows/runner/flutter_window.cpp +143 -32
  16. package/landing/index.html +3 -1
  17. package/lib/manager.js +106 -89
  18. package/lib/schema_migrations.js +115 -13
  19. package/package.json +30 -15
  20. package/runtime/paths.js +49 -5
  21. package/server/db/database.js +2 -2
  22. package/server/guest-agent.cli.package.json +13 -0
  23. package/server/guest_agent.js +85 -40
  24. package/server/http/middleware.js +24 -0
  25. package/server/http/routes.js +11 -6
  26. package/server/public/.last_build_id +1 -1
  27. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  28. package/server/public/flutter_bootstrap.js +2 -2
  29. package/server/public/main.dart.js +73083 -72209
  30. package/server/routes/admin.js +1 -1
  31. package/server/routes/android.js +30 -34
  32. package/server/routes/browser.js +23 -15
  33. package/server/routes/desktop.js +18 -1
  34. package/server/routes/integrations.js +107 -1
  35. package/server/routes/memory.js +1 -0
  36. package/server/routes/settings.js +16 -5
  37. package/server/routes/social_reach.js +12 -3
  38. package/server/routes/social_video.js +4 -0
  39. package/server/services/agents/manager.js +1 -1
  40. package/server/services/ai/capabilityHealth.js +62 -96
  41. package/server/services/ai/compaction.js +7 -2
  42. package/server/services/ai/history.js +45 -6
  43. package/server/services/ai/integrated_tools/http_request.js +8 -0
  44. package/server/services/ai/loop/agent_engine_core.js +390 -162
  45. package/server/services/ai/loop/blank_recovery.js +5 -4
  46. package/server/services/ai/loop/callbacks.js +1 -0
  47. package/server/services/ai/loop/completion_judge.js +121 -5
  48. package/server/services/ai/loop/conversation_loop.js +510 -332
  49. package/server/services/ai/loop/messaging_delivery.js +129 -57
  50. package/server/services/ai/loop/model_call_guard.js +91 -0
  51. package/server/services/ai/loop/model_io.js +20 -45
  52. package/server/services/ai/loop/progress_classification.js +2 -0
  53. package/server/services/ai/loop/tool_dispatch.js +19 -8
  54. package/server/services/ai/loopPolicy.js +48 -21
  55. package/server/services/ai/messagingFallback.js +17 -17
  56. package/server/services/ai/model_discovery.js +227 -0
  57. package/server/services/ai/model_failure_cache.js +108 -0
  58. package/server/services/ai/model_identity.js +71 -0
  59. package/server/services/ai/models.js +68 -163
  60. package/server/services/ai/providerRetry.js +17 -59
  61. package/server/services/ai/provider_selector.js +166 -0
  62. package/server/services/ai/providers/anthropic.js +2 -2
  63. package/server/services/ai/providers/claudeCode.js +21 -33
  64. package/server/services/ai/providers/githubCopilot.js +41 -20
  65. package/server/services/ai/providers/google.js +135 -97
  66. package/server/services/ai/providers/grok.js +4 -3
  67. package/server/services/ai/providers/grokOauth.js +19 -27
  68. package/server/services/ai/providers/nvidia.js +10 -5
  69. package/server/services/ai/providers/ollama.js +111 -84
  70. package/server/services/ai/providers/ollama_stream.js +142 -0
  71. package/server/services/ai/providers/openai.js +39 -5
  72. package/server/services/ai/providers/openaiCodex.js +11 -4
  73. package/server/services/ai/providers/openrouter.js +29 -7
  74. package/server/services/ai/providers/provider_error.js +36 -0
  75. package/server/services/ai/settings.js +26 -2
  76. package/server/services/ai/systemPrompt.js +19 -12
  77. package/server/services/ai/taskAnalysis.js +58 -10
  78. package/server/services/ai/terminal_reply.js +18 -0
  79. package/server/services/ai/toolEvidence.js +350 -29
  80. package/server/services/ai/tools.js +190 -111
  81. package/server/services/android/controller.js +770 -237
  82. package/server/services/android/process.js +140 -0
  83. package/server/services/android/sdk_download.js +143 -0
  84. package/server/services/android/uia.js +6 -5
  85. package/server/services/artifacts/store.js +24 -0
  86. package/server/services/browser/controller.js +843 -385
  87. package/server/services/browser/extension/gateway.js +40 -16
  88. package/server/services/browser/extension/protocol.js +15 -1
  89. package/server/services/browser/extension/provider.js +71 -47
  90. package/server/services/browser/extension/registry.js +155 -34
  91. package/server/services/cli/executor.js +62 -9
  92. package/server/services/credentials/bitwarden_cli.js +322 -0
  93. package/server/services/credentials/broker.js +594 -0
  94. package/server/services/desktop/gateway.js +41 -4
  95. package/server/services/desktop/protocol.js +3 -0
  96. package/server/services/desktop/provider.js +39 -42
  97. package/server/services/desktop/registry.js +137 -52
  98. package/server/services/integrations/bitwarden/constants.js +14 -0
  99. package/server/services/integrations/bitwarden/provider.js +197 -0
  100. package/server/services/integrations/bitwarden/snapshot.js +65 -0
  101. package/server/services/integrations/figma/provider.js +78 -12
  102. package/server/services/integrations/github/common.js +11 -6
  103. package/server/services/integrations/github/provider.js +52 -53
  104. package/server/services/integrations/google/provider.js +55 -19
  105. package/server/services/integrations/home_assistant/network.js +17 -20
  106. package/server/services/integrations/home_assistant/provider.js +7 -5
  107. package/server/services/integrations/home_assistant/tools.js +17 -5
  108. package/server/services/integrations/http.js +51 -0
  109. package/server/services/integrations/manager.js +159 -53
  110. package/server/services/integrations/microsoft/provider.js +80 -13
  111. package/server/services/integrations/neoarchive/provider.js +55 -29
  112. package/server/services/integrations/neorecall/client.js +17 -10
  113. package/server/services/integrations/neorecall/provider.js +20 -11
  114. package/server/services/integrations/notion/provider.js +16 -13
  115. package/server/services/integrations/oauth_provider.js +115 -51
  116. package/server/services/integrations/registry.js +2 -0
  117. package/server/services/integrations/slack/provider.js +98 -9
  118. package/server/services/integrations/spotify/provider.js +67 -71
  119. package/server/services/integrations/trello/provider.js +21 -7
  120. package/server/services/integrations/weather/provider.js +18 -12
  121. package/server/services/integrations/whatsapp/provider.js +76 -16
  122. package/server/services/manager.js +110 -1
  123. package/server/services/memory/embedding_index.js +20 -8
  124. package/server/services/memory/embeddings.js +151 -90
  125. package/server/services/memory/ingestion.js +50 -9
  126. package/server/services/memory/ingestion_documents.js +13 -3
  127. package/server/services/memory/manager.js +52 -19
  128. package/server/services/messaging/automation.js +85 -10
  129. package/server/services/messaging/formatting_guides.js +7 -4
  130. package/server/services/messaging/http_platforms.js +33 -13
  131. package/server/services/messaging/inbound_queue.js +78 -24
  132. package/server/services/messaging/inbound_store.js +224 -0
  133. package/server/services/messaging/manager.js +326 -51
  134. package/server/services/messaging/typing_keepalive.js +5 -2
  135. package/server/services/messaging/whatsapp.js +22 -14
  136. package/server/services/network/http.js +210 -0
  137. package/server/services/network/safe_request.js +307 -0
  138. package/server/services/runtime/backends/local-vm.js +227 -67
  139. package/server/services/runtime/docker-vm-manager.js +9 -0
  140. package/server/services/runtime/guest_bootstrap.js +30 -4
  141. package/server/services/runtime/guest_image.js +43 -12
  142. package/server/services/runtime/manager.js +77 -23
  143. package/server/services/runtime/validation.js +7 -6
  144. package/server/services/security/tool_categories.js +6 -0
  145. package/server/services/social_reach/channels/github.js +10 -4
  146. package/server/services/social_reach/channels/reddit.js +4 -4
  147. package/server/services/social_reach/channels/rss.js +2 -2
  148. package/server/services/social_reach/channels/social_video.js +12 -7
  149. package/server/services/social_reach/channels/v2ex.js +21 -8
  150. package/server/services/social_reach/channels/x.js +2 -2
  151. package/server/services/social_reach/channels/xueqiu.js +5 -5
  152. package/server/services/social_reach/service.js +9 -6
  153. package/server/services/social_reach/utils.js +65 -14
  154. package/server/services/social_video/service.js +160 -50
  155. package/server/services/tasks/integration_runtime.js +18 -8
  156. package/server/services/tasks/runtime.js +39 -4
  157. package/server/services/voice/agentBridge.js +17 -4
  158. package/server/services/voice/bufferedLiveRelayAdapter.js +5 -0
  159. package/server/services/voice/liveSession.js +31 -0
  160. package/server/services/voice/message.js +1 -1
  161. package/server/services/voice/openaiSpeech.js +33 -8
  162. package/server/services/voice/providers.js +233 -151
  163. package/server/services/voice/runtime.js +2 -2
  164. package/server/services/voice/runtimeManager.js +118 -20
  165. package/server/services/voice/turnRunner.js +6 -0
  166. package/server/services/wearable/firmware_manifest.js +51 -13
  167. package/server/services/wearable/service.js +1 -0
  168. package/server/utils/abort.js +96 -0
  169. package/server/utils/cloud-security.js +110 -3
  170. package/server/utils/files.js +31 -0
  171. package/server/utils/image_payload.js +95 -0
  172. package/server/utils/retry.js +107 -0
@@ -3,6 +3,7 @@ const { BROWSER_EXTENSION_WS_PATH } = require('./protocol');
3
3
 
4
4
  const DEFAULT_UPGRADE_RATE_LIMIT_WINDOW_MS = 60 * 1000;
5
5
  const DEFAULT_UPGRADE_RATE_LIMIT_MAX = 30;
6
+ const MAX_EXTENSION_MESSAGE_BYTES = 32 * 1024 * 1024;
6
7
 
7
8
  function rejectUpgrade(socket, statusCode, message) {
8
9
  try {
@@ -22,8 +23,13 @@ function rejectUpgrade(socket, statusCode, message) {
22
23
  }
23
24
 
24
25
  function bindBrowserExtensionGateway(httpServer, app) {
25
- const wss = new WebSocketServer({ noServer: true });
26
+ const wss = new WebSocketServer({
27
+ noServer: true,
28
+ maxPayload: MAX_EXTENSION_MESSAGE_BYTES,
29
+ });
26
30
  const attemptsByIp = new Map();
31
+ let closing = false;
32
+ let closePromise = null;
27
33
  const windowMs = Number(process.env.NEOAGENT_BROWSER_EXTENSION_UPGRADE_WINDOW_MS || DEFAULT_UPGRADE_RATE_LIMIT_WINDOW_MS);
28
34
 
29
35
  const cleanupTimer = setInterval(() => {
@@ -50,7 +56,7 @@ function bindBrowserExtensionGateway(httpServer, app) {
50
56
  return entry.count > maxAttempts;
51
57
  }
52
58
 
53
- httpServer.on('upgrade', (req, socket, head) => {
59
+ const handleUpgrade = (req, socket, head) => {
54
60
  let url;
55
61
  try {
56
62
  url = new URL(req.url, 'http://localhost');
@@ -60,6 +66,10 @@ function bindBrowserExtensionGateway(httpServer, app) {
60
66
  if (url.pathname !== BROWSER_EXTENSION_WS_PATH) {
61
67
  return;
62
68
  }
69
+ if (closing) {
70
+ rejectUpgrade(socket, 503, 'Service Unavailable');
71
+ return;
72
+ }
63
73
 
64
74
  const remoteAddress = req.socket?.remoteAddress || 'unknown';
65
75
  if (isRateLimited(remoteAddress)) {
@@ -81,24 +91,38 @@ function bindBrowserExtensionGateway(httpServer, app) {
81
91
  }
82
92
 
83
93
  wss.handleUpgrade(req, socket, head, (ws) => {
84
- registry.registerConnection(tokenRow, ws, {
85
- remoteAddress,
86
- userAgent: req.headers['user-agent'] || null,
87
- });
88
- ws.send(JSON.stringify({
89
- type: 'hello',
90
- ok: true,
91
- userId: tokenRow.user_id,
92
- tokenId: tokenRow.id,
93
- }));
94
+ try {
95
+ registry.registerConnection(tokenRow, ws, {
96
+ remoteAddress,
97
+ userAgent: req.headers['user-agent'] || null,
98
+ });
99
+ ws.send(JSON.stringify({
100
+ type: 'hello',
101
+ ok: true,
102
+ userId: tokenRow.user_id,
103
+ tokenId: tokenRow.id,
104
+ }));
105
+ } catch (error) {
106
+ try { ws.close(1012, String(error?.message || 'Service unavailable').slice(0, 120)); } catch {}
107
+ }
94
108
  });
95
- });
109
+ };
110
+ httpServer.on('upgrade', handleUpgrade);
96
111
 
97
112
  app.locals.browserExtensionGateway = {
98
- close: () => new Promise((resolve) => {
113
+ close: () => {
114
+ if (closePromise) return closePromise;
115
+ closing = true;
99
116
  clearInterval(cleanupTimer);
100
- wss.close(() => resolve());
101
- }),
117
+ httpServer.removeListener('upgrade', handleUpgrade);
118
+ for (const client of wss.clients) {
119
+ try { client.terminate(); } catch {}
120
+ }
121
+ closePromise = new Promise((resolve) => {
122
+ wss.close(() => resolve());
123
+ });
124
+ return closePromise;
125
+ },
102
126
  };
103
127
 
104
128
  return wss;
@@ -1,6 +1,15 @@
1
+ 'use strict';
2
+
1
3
  const BROWSER_EXTENSION_WS_PATH = '/api/browser-extension/ws';
2
4
  const EXTENSION_PROTOCOL_VERSION = 1;
3
5
 
6
+ const EXTENSION_MESSAGE_TYPES = Object.freeze({
7
+ COMMAND: 'command',
8
+ RESULT: 'result',
9
+ URL_VALIDATION_REQUEST: 'urlValidationRequest',
10
+ URL_VALIDATION_RESULT: 'urlValidationResult',
11
+ });
12
+
4
13
  const EXTENSION_COMMANDS = Object.freeze({
5
14
  LAUNCH: 'launch',
6
15
  NAVIGATE: 'navigate',
@@ -16,6 +25,10 @@ const EXTENSION_COMMANDS = Object.freeze({
16
25
  CLOSE: 'close',
17
26
  GET_PAGE_INFO: 'getPageInfo',
18
27
  GET_COOKIES: 'getCookies',
28
+ FILL_CREDENTIAL: 'fillCredential',
29
+ SUBMIT_CREDENTIAL: 'submitCredential',
30
+ CANCEL_CREDENTIAL: 'cancelCredential',
31
+ CANCEL_COMMAND: 'cancelCommand',
19
32
  });
20
33
 
21
34
  class ExtensionBrowserUnavailableError extends Error {
@@ -28,7 +41,7 @@ class ExtensionBrowserUnavailableError extends Error {
28
41
 
29
42
  function createCommandMessage(id, command, payload = {}) {
30
43
  return {
31
- type: 'command',
44
+ type: EXTENSION_MESSAGE_TYPES.COMMAND,
32
45
  version: EXTENSION_PROTOCOL_VERSION,
33
46
  id,
34
47
  command,
@@ -46,6 +59,7 @@ function parseExtensionMessage(data) {
46
59
  module.exports = {
47
60
  BROWSER_EXTENSION_WS_PATH,
48
61
  EXTENSION_PROTOCOL_VERSION,
62
+ EXTENSION_MESSAGE_TYPES,
49
63
  EXTENSION_COMMANDS,
50
64
  ExtensionBrowserUnavailableError,
51
65
  createCommandMessage,
@@ -1,18 +1,15 @@
1
+ 'use strict';
2
+
1
3
  const fs = require('fs');
2
4
  const path = require('path');
3
5
  const { DATA_DIR } = require('../../../../runtime/paths');
6
+ const { writeBufferAtomic } = require('../../../utils/files');
7
+ const { decodeBase64Image } = require('../../../utils/image_payload');
4
8
  const { EXTENSION_COMMANDS, ExtensionBrowserUnavailableError } = require('./protocol');
5
9
 
6
10
  const SCREENSHOTS_DIR = path.join(DATA_DIR, 'screenshots');
7
11
  if (!fs.existsSync(SCREENSHOTS_DIR)) fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
8
12
 
9
- function extractBase64Png(value) {
10
- const text = String(value || '');
11
- if (!text) return null;
12
- const match = text.match(/^data:image\/png;base64,(.+)$/);
13
- return match ? match[1] : text;
14
- }
15
-
16
13
  class ExtensionBrowserProvider {
17
14
  constructor(options = {}) {
18
15
  this.registry = options.registry;
@@ -31,11 +28,18 @@ class ExtensionBrowserProvider {
31
28
 
32
29
  async #dispatch(command, payload = {}, options = {}) {
33
30
  this.#assertReady();
34
- const result = await this.registry.dispatch(this.userId, command, payload, {
31
+ const safePayload = { ...(payload || {}) };
32
+ const signal = options.signal || safePayload.signal || null;
33
+ const timeoutMs = options.timeoutMs ?? safePayload.timeoutMs;
34
+ delete safePayload.signal;
35
+ delete safePayload.timeoutMs;
36
+ const result = await this.registry.dispatch(this.userId, command, safePayload, {
35
37
  ...options,
38
+ signal,
39
+ timeoutMs,
36
40
  tokenId: options.tokenId || this.tokenId,
37
41
  });
38
- return this.#materialize(result);
42
+ return this.#materialize(result, { signal });
39
43
  }
40
44
 
41
45
  #disconnect() {
@@ -46,18 +50,17 @@ class ExtensionBrowserProvider {
46
50
  }
47
51
  }
48
52
 
49
- #writeScreenshotArtifact(base64) {
50
- const buffer = Buffer.from(base64, 'base64');
53
+ async #writeScreenshotArtifact(image, options = {}) {
51
54
  if (this.artifactStore && this.userId != null) {
52
- const artifact = this.artifactStore.allocateFile(this.userId, {
55
+ const artifact = await this.artifactStore.createBufferArtifact(this.userId, {
53
56
  kind: 'browser-screenshot',
54
57
  backend: 'extension',
55
- extension: 'png',
56
- contentType: 'image/png',
58
+ extension: image.extension,
59
+ contentType: image.contentType,
57
60
  filenameBase: 'browser-extension-screenshot',
61
+ content: image.buffer,
62
+ signal: options.signal,
58
63
  });
59
- fs.writeFileSync(artifact.storagePath, buffer);
60
- this.artifactStore.finalizeFile(artifact.artifactId, artifact.storagePath);
61
64
  return {
62
65
  screenshotPath: artifact.url,
63
66
  artifactId: artifact.artifactId,
@@ -68,7 +71,7 @@ class ExtensionBrowserProvider {
68
71
 
69
72
  const filename = `browser_extension_${Date.now()}_${Math.random().toString(16).slice(2)}.png`;
70
73
  const fullPath = path.join(SCREENSHOTS_DIR, filename);
71
- fs.writeFileSync(fullPath, buffer);
74
+ await writeBufferAtomic(fullPath, image.buffer, { signal: options.signal });
72
75
  return {
73
76
  screenshotPath: `/screenshots/${filename}`,
74
77
  artifactId: null,
@@ -77,13 +80,12 @@ class ExtensionBrowserProvider {
77
80
  };
78
81
  }
79
82
 
80
- #materialize(result) {
83
+ async #materialize(result, options = {}) {
81
84
  if (!result || typeof result !== 'object') return result;
82
85
  const raw = result.screenshotDataUrl || result.screenshotData || result.screenshotBase64;
83
86
  if (!raw) return result;
84
- const base64 = extractBase64Png(raw);
85
- if (!base64) return result;
86
- const screenshot = this.#writeScreenshotArtifact(base64);
87
+ const image = decodeBase64Image(raw, { allowedTypes: ['image/png'] });
88
+ const screenshot = await this.#writeScreenshotArtifact(image, options);
87
89
  const next = { ...result, ...screenshot };
88
90
  delete next.screenshotDataUrl;
89
91
  delete next.screenshotData;
@@ -92,79 +94,102 @@ class ExtensionBrowserProvider {
92
94
  }
93
95
 
94
96
  navigate(url, options = {}) {
95
- return this.#dispatch(EXTENSION_COMMANDS.NAVIGATE, { url, ...options });
97
+ return this.#dispatch(EXTENSION_COMMANDS.NAVIGATE, { url, ...options }, options);
96
98
  }
97
99
 
98
- click(selector, text, screenshot = true) {
99
- return this.#dispatch(EXTENSION_COMMANDS.CLICK, { selector, text, screenshot });
100
+ click(selector, text, screenshot = true, options = {}) {
101
+ return this.#dispatch(EXTENSION_COMMANDS.CLICK, { selector, text, screenshot }, options);
100
102
  }
101
103
 
102
- clickPoint(x, y, screenshot = true) {
103
- return this.#dispatch(EXTENSION_COMMANDS.CLICK_POINT, { x, y, screenshot });
104
+ clickPoint(x, y, screenshot = true, options = {}) {
105
+ return this.#dispatch(EXTENSION_COMMANDS.CLICK_POINT, { x, y, screenshot }, options);
104
106
  }
105
107
 
106
108
  type(selector, text, options = {}) {
107
- return this.#dispatch(EXTENSION_COMMANDS.TYPE, { selector, text, ...options });
109
+ return this.#dispatch(EXTENSION_COMMANDS.TYPE, { selector, text, ...options }, options);
108
110
  }
109
111
 
110
112
  typeText(text, options = {}) {
111
- return this.#dispatch(EXTENSION_COMMANDS.TYPE_TEXT, { text, ...options });
113
+ return this.#dispatch(EXTENSION_COMMANDS.TYPE_TEXT, { text, ...options }, options);
112
114
  }
113
115
 
114
- pressKey(key, screenshot = true) {
115
- return this.#dispatch(EXTENSION_COMMANDS.PRESS_KEY, { key, screenshot });
116
+ pressKey(key, screenshot = true, options = {}) {
117
+ return this.#dispatch(EXTENSION_COMMANDS.PRESS_KEY, { key, screenshot }, options);
116
118
  }
117
119
 
118
- scroll(deltaX = 0, deltaY = 0, screenshot = true) {
119
- return this.#dispatch(EXTENSION_COMMANDS.SCROLL, { deltaX, deltaY, screenshot });
120
+ scroll(deltaX = 0, deltaY = 0, screenshot = true, options = {}) {
121
+ return this.#dispatch(EXTENSION_COMMANDS.SCROLL, { deltaX, deltaY, screenshot }, options);
120
122
  }
121
123
 
122
- extract(selector, attribute, all = false) {
123
- return this.#dispatch(EXTENSION_COMMANDS.EXTRACT, { selector, attribute, all });
124
+ extract(selector, attribute, all = false, options = {}) {
125
+ return this.#dispatch(EXTENSION_COMMANDS.EXTRACT, { selector, attribute, all }, options);
124
126
  }
125
127
 
126
- evaluate(script) {
127
- return this.#dispatch(EXTENSION_COMMANDS.EVALUATE, { script });
128
+ evaluate(script, options = {}) {
129
+ return this.#dispatch(EXTENSION_COMMANDS.EVALUATE, { script }, options);
128
130
  }
129
131
 
130
132
  screenshot(options = {}) {
131
- return this.#dispatch(EXTENSION_COMMANDS.SCREENSHOT, options);
133
+ return this.#dispatch(EXTENSION_COMMANDS.SCREENSHOT, options, options);
132
134
  }
133
135
 
134
136
  launch(options = {}) {
135
- return this.#dispatch(EXTENSION_COMMANDS.LAUNCH, options);
137
+ return this.#dispatch(EXTENSION_COMMANDS.LAUNCH, options, options);
136
138
  }
137
139
 
138
- async closeBrowser() {
140
+ async closeBrowser(options = {}) {
139
141
  if (!this.registry || this.userId == null || !this.registry.isConnected(this.userId, this.tokenId)) {
140
142
  return { success: true, extensionConnected: false };
141
143
  }
142
- const result = await this.#dispatch(EXTENSION_COMMANDS.CLOSE, {});
144
+ const result = await this.#dispatch(EXTENSION_COMMANDS.CLOSE, {}, options);
143
145
  this.#disconnect();
144
146
  return { ...result, success: result?.success !== false, extensionConnected: false };
145
147
  }
146
148
 
147
- fill(selector, value) {
148
- return this.type(selector, String(value));
149
+ fill(selector, value, options = {}) {
150
+ return this.type(selector, String(value), options);
151
+ }
152
+
153
+ fillCredential(input, options = {}) {
154
+ return this.#dispatch(EXTENSION_COMMANDS.FILL_CREDENTIAL, input, options);
155
+ }
156
+
157
+ submitProtectedCredential(protectedFillId, options = {}) {
158
+ return this.#dispatch(EXTENSION_COMMANDS.SUBMIT_CREDENTIAL, { protectedFillId }, options);
159
+ }
160
+
161
+ cancelProtectedCredential(protectedFillId, options = {}) {
162
+ return this.#dispatch(EXTENSION_COMMANDS.CANCEL_CREDENTIAL, { protectedFillId }, options);
149
163
  }
150
164
 
151
165
  extractContent(options = {}) {
152
- return this.extract(options.selector, options.attribute, options.all);
166
+ return this.extract(options.selector, options.attribute, options.all, options);
153
167
  }
154
168
 
155
- executeJS(code) {
156
- return this.evaluate(code);
169
+ executeJS(code, options = {}) {
170
+ return this.evaluate(code, options);
157
171
  }
158
172
 
159
- async getPageInfo() {
173
+ async getPageInfo(options = {}) {
160
174
  if (!this.registry || this.userId == null || !this.registry.isConnected(this.userId, this.tokenId)) {
161
175
  return { url: null, title: null, extensionConnected: false };
162
176
  }
163
177
  return this.registry.dispatch(this.userId, EXTENSION_COMMANDS.GET_PAGE_INFO, {}, {
164
178
  tokenId: this.tokenId,
179
+ signal: options.signal,
165
180
  });
166
181
  }
167
182
 
183
+ async getCookies(options = {}) {
184
+ const pageInfo = await this.getPageInfo(options);
185
+ let hostname = '';
186
+ try { hostname = new URL(String(pageInfo?.url || '')).hostname; } catch {}
187
+ if (!hostname) return { cookies: [], domains: [] };
188
+ return this.#dispatch(EXTENSION_COMMANDS.GET_COOKIES, {
189
+ domains: [hostname],
190
+ }, options);
191
+ }
192
+
168
193
  isLaunched() {
169
194
  return Boolean(this.registry && this.userId != null && this.registry.isConnected(this.userId, this.tokenId));
170
195
  }
@@ -181,5 +206,4 @@ class ExtensionBrowserProvider {
181
206
 
182
207
  module.exports = {
183
208
  ExtensionBrowserProvider,
184
- extractBase64Png,
185
209
  };
@@ -1,6 +1,13 @@
1
+ 'use strict';
2
+
1
3
  const crypto = require('crypto');
2
4
  const db = require('../../../db/database');
5
+ const { createAbortError } = require('../../../utils/abort');
6
+ const { validateCloudUrlWithDns } = require('../../../utils/cloud-security');
3
7
  const {
8
+ EXTENSION_COMMANDS,
9
+ EXTENSION_MESSAGE_TYPES,
10
+ EXTENSION_PROTOCOL_VERSION,
4
11
  ExtensionBrowserUnavailableError,
5
12
  createCommandMessage,
6
13
  parseExtensionMessage,
@@ -11,6 +18,8 @@ const DEFAULT_COMMAND_TIMEOUT_MS = 30 * 1000;
11
18
  const DEFAULT_HEARTBEAT_INTERVAL_MS = 25 * 1000;
12
19
  const DEFAULT_HEARTBEAT_TIMEOUT_MS = 75 * 1000;
13
20
  const DEFAULT_PRESENCE_TOUCH_INTERVAL_MS = 15 * 1000;
21
+ const DEFAULT_URL_VALIDATION_LIMIT = 128;
22
+ const MAX_URL_VALIDATION_CHARS = 8192;
14
23
 
15
24
  function sha256(value) {
16
25
  return crypto.createHash('sha256').update(String(value || '')).digest('hex');
@@ -48,7 +57,13 @@ class BrowserExtensionRegistry {
48
57
  this.heartbeatIntervalMs = Number(options.heartbeatIntervalMs || process.env.NEOAGENT_BROWSER_EXTENSION_HEARTBEAT_INTERVAL_MS || DEFAULT_HEARTBEAT_INTERVAL_MS);
49
58
  this.heartbeatTimeoutMs = Number(options.heartbeatTimeoutMs || process.env.NEOAGENT_BROWSER_EXTENSION_HEARTBEAT_TIMEOUT_MS || DEFAULT_HEARTBEAT_TIMEOUT_MS);
50
59
  this.presenceTouchIntervalMs = Number(options.presenceTouchIntervalMs || process.env.NEOAGENT_BROWSER_EXTENSION_PRESENCE_TOUCH_INTERVAL_MS || DEFAULT_PRESENCE_TOUCH_INTERVAL_MS);
60
+ this.urlValidator = options.urlValidator || validateCloudUrlWithDns;
61
+ this.urlValidationLimit = Math.max(1, Math.min(
62
+ 512,
63
+ Number(options.urlValidationLimit || DEFAULT_URL_VALIDATION_LIMIT),
64
+ ));
51
65
  this.connectionsByUser = new Map();
66
+ this.closed = false;
52
67
  }
53
68
 
54
69
  #getUserConnections(userId, create = false) {
@@ -204,6 +219,9 @@ class BrowserExtensionRegistry {
204
219
  }
205
220
 
206
221
  registerConnection(tokenRow, ws, meta = {}) {
222
+ if (this.closed) {
223
+ throw new ExtensionBrowserUnavailableError('Browser extension registry is shutting down.');
224
+ }
207
225
  const userId = String(tokenRow.user_id);
208
226
  const userMap = this.#getUserConnections(userId, true);
209
227
  const existing = userMap.get(tokenRow.id);
@@ -223,11 +241,15 @@ class BrowserExtensionRegistry {
223
241
  presenceTouchIntervalMs: this.presenceTouchIntervalMs,
224
242
  });
225
243
  userMap.set(tokenRow.id, connection);
226
- this.db.prepare(
227
- `UPDATE browser_extension_tokens
228
- SET last_connected_at = datetime('now'), last_seen_at = datetime('now')
229
- WHERE id = ?`
230
- ).run(tokenRow.id);
244
+ try {
245
+ this.db.prepare(
246
+ `UPDATE browser_extension_tokens
247
+ SET last_connected_at = datetime('now'), last_seen_at = datetime('now')
248
+ WHERE id = ?`
249
+ ).run(tokenRow.id);
250
+ } catch (error) {
251
+ console.warn('[BrowserExtension] Failed to record connection presence:', error?.message);
252
+ }
231
253
  return connection;
232
254
  }
233
255
 
@@ -322,9 +344,13 @@ class BrowserExtensionRegistry {
322
344
  const userMap = this.#getUserConnections(userId);
323
345
  const connection = userMap?.get(String(tokenId));
324
346
  if (!connection?.isOpen()) return;
325
- this.db.prepare(
326
- `UPDATE browser_extension_tokens SET last_seen_at = datetime('now') WHERE id = ?`
327
- ).run(tokenId);
347
+ try {
348
+ this.db.prepare(
349
+ `UPDATE browser_extension_tokens SET last_seen_at = datetime('now') WHERE id = ?`
350
+ ).run(tokenId);
351
+ } catch (error) {
352
+ console.warn('[BrowserExtension] Failed to update connection presence:', error?.message);
353
+ }
328
354
  }
329
355
 
330
356
  async dispatch(userId, command, payload = {}, options = {}) {
@@ -333,12 +359,14 @@ class BrowserExtensionRegistry {
333
359
  throw new ExtensionBrowserUnavailableError();
334
360
  }
335
361
  const result = await connection.sendCommand(command, payload, options);
336
- this.db.prepare(
337
- `UPDATE browser_extension_tokens SET last_seen_at = datetime('now') WHERE id = ?`
338
- ).run(connection.tokenId);
362
+ this.touchPresence(userId, connection.tokenId);
339
363
  return result;
340
364
  }
341
365
 
366
+ validateBrowserUrl(url, options = {}) {
367
+ return this.urlValidator(url, options);
368
+ }
369
+
342
370
  getStatus(userId) {
343
371
  const userMap = this.#getUserConnections(userId);
344
372
  let selectedTokenId = this.getSelectedTokenId(userId);
@@ -396,22 +424,22 @@ class BrowserExtensionRegistry {
396
424
  ).run(userId);
397
425
  }
398
426
 
399
- const connection = this.getConnection(userId, targetTokenId);
400
- if (connection && (!targetTokenId || connection.tokenId === targetTokenId)) {
427
+ const userMap = this.#getUserConnections(userId);
428
+ const connections = targetTokenId
429
+ ? [userMap?.get(targetTokenId)].filter(Boolean)
430
+ : Array.from(userMap?.values() || []);
431
+ for (const connection of connections) {
401
432
  connection.close('extension token revoked');
402
433
  }
403
434
  return { success: true };
404
435
  }
405
436
 
406
437
  closeAll() {
407
- for (const connection of this.connectionsByUser.values()) {
408
- if (connection instanceof Map) {
409
- for (const nested of connection.values()) {
410
- nested.close('server shutdown');
411
- }
412
- } else {
413
- connection.close('server shutdown');
414
- }
438
+ this.closed = true;
439
+ const connections = Array.from(this.connectionsByUser.values())
440
+ .flatMap((entry) => entry instanceof Map ? Array.from(entry.values()) : [entry]);
441
+ for (const connection of connections) {
442
+ connection.close('server shutdown');
415
443
  }
416
444
  this.connectionsByUser.clear();
417
445
  }
@@ -433,24 +461,34 @@ class ExtensionBrowserConnection {
433
461
  this.lastPongAt = Date.now();
434
462
  this.lastPresenceTouchAt = 0;
435
463
  this.heartbeatTimer = null;
464
+ this.urlValidationCount = 0;
465
+ this.urlValidationController = new AbortController();
466
+ this.closed = false;
436
467
 
437
- ws.on('message', (data) => this.#handleMessage(data));
468
+ ws.on('message', (data) => {
469
+ this.#handleMessage(data).catch(() => {});
470
+ });
438
471
  ws.on('pong', () => {
439
472
  this.lastPongAt = Date.now();
440
473
  this.touchPresence();
441
474
  });
442
475
  ws.on('close', () => this.#closePending(new ExtensionBrowserUnavailableError('Extension browser disconnected.')));
443
- ws.on('error', (error) => this.#closePending(error));
476
+ ws.on('error', (error) => {
477
+ try { ws.terminate(); } catch {}
478
+ this.#closePending(error);
479
+ });
444
480
  this.#startHeartbeat();
445
481
  }
446
482
 
447
483
  isOpen() {
448
- return this.ws && this.ws.readyState === 1;
484
+ return !this.closed && this.ws && this.ws.readyState === 1;
449
485
  }
450
486
 
451
487
  close(reason) {
488
+ const wasOpen = this.isOpen();
489
+ this.closed = true;
452
490
  try {
453
- if (this.isOpen()) {
491
+ if (wasOpen) {
454
492
  this.ws.close(1000, String(reason || 'closing').slice(0, 120));
455
493
  }
456
494
  } catch {}
@@ -478,25 +516,57 @@ class ExtensionBrowserConnection {
478
516
  return Promise.reject(new ExtensionBrowserUnavailableError());
479
517
  }
480
518
  const id = crypto.randomUUID();
481
- const timeoutMs = Number(options.timeoutMs || this.timeoutMs);
519
+ const requestedTimeout = Number(options.timeoutMs || this.timeoutMs);
520
+ const timeoutMs = Number.isFinite(requestedTimeout)
521
+ ? Math.min(10 * 60 * 1000, Math.max(100, requestedTimeout))
522
+ : DEFAULT_COMMAND_TIMEOUT_MS;
482
523
  const message = createCommandMessage(id, command, payload);
483
524
  return new Promise((resolve, reject) => {
484
- const timer = setTimeout(() => {
525
+ const signal = options.signal || null;
526
+ const sendCancellation = () => {
527
+ if (!this.isOpen() || command === EXTENSION_COMMANDS.CANCEL_COMMAND) return;
528
+ try {
529
+ this.ws.send(JSON.stringify(createCommandMessage(
530
+ crypto.randomUUID(),
531
+ EXTENSION_COMMANDS.CANCEL_COMMAND,
532
+ { commandId: id },
533
+ )));
534
+ } catch {}
535
+ };
536
+ const cleanup = () => {
537
+ clearTimeout(timer);
538
+ signal?.removeEventListener('abort', onAbort);
485
539
  this.pending.delete(id);
486
- reject(new Error(`Browser extension command timed out: ${command}`));
540
+ };
541
+ const onAbort = () => {
542
+ cleanup();
543
+ sendCancellation();
544
+ reject(createAbortError(signal, `Browser extension command aborted: ${command}`));
545
+ };
546
+ const timer = setTimeout(() => {
547
+ cleanup();
548
+ sendCancellation();
549
+ const error = new Error(`Browser extension command timed out: ${command}`);
550
+ error.code = 'BROWSER_EXTENSION_COMMAND_TIMEOUT';
551
+ reject(error);
487
552
  }, timeoutMs);
488
- this.pending.set(id, { resolve, reject, timer, command });
553
+ timer.unref?.();
554
+ if (signal?.aborted) {
555
+ onAbort();
556
+ return;
557
+ }
558
+ signal?.addEventListener('abort', onAbort, { once: true });
559
+ this.pending.set(id, { resolve, reject, timer, command, signal, onAbort });
489
560
  try {
490
561
  this.ws.send(JSON.stringify(message));
491
562
  } catch (error) {
492
- clearTimeout(timer);
493
- this.pending.delete(id);
563
+ cleanup();
494
564
  reject(error);
495
565
  }
496
566
  });
497
567
  }
498
568
 
499
- #handleMessage(data) {
569
+ async #handleMessage(data) {
500
570
  this.lastPongAt = Date.now();
501
571
  this.touchPresence();
502
572
  let message;
@@ -505,21 +575,70 @@ class ExtensionBrowserConnection {
505
575
  } catch {
506
576
  return;
507
577
  }
508
- if (!message || message.type !== 'result' || !message.id) {
578
+ if (!message || !message.id) {
509
579
  return;
510
580
  }
581
+ if (message.type === EXTENSION_MESSAGE_TYPES.URL_VALIDATION_REQUEST) {
582
+ await this.#handleUrlValidationRequest(message);
583
+ return;
584
+ }
585
+ if (message.type !== EXTENSION_MESSAGE_TYPES.RESULT) return;
511
586
  const pending = this.pending.get(message.id);
512
587
  if (!pending) return;
513
588
  clearTimeout(pending.timer);
589
+ pending.signal?.removeEventListener('abort', pending.onAbort);
514
590
  this.pending.delete(message.id);
515
591
  if (message.ok === false) {
516
592
  pending.reject(new Error(message.error || `Browser extension command failed: ${pending.command}`));
517
593
  return;
518
594
  }
519
- pending.resolve(message.result || {});
595
+ pending.resolve(message.result ?? {});
596
+ }
597
+
598
+ async #handleUrlValidationRequest(message) {
599
+ const id = typeof message.id === 'string' ? message.id : '';
600
+ if (!id || id.length > 200 || !this.isOpen()) return;
601
+
602
+ let allowed = false;
603
+ const url = typeof message.url === 'string' ? message.url : '';
604
+ const versionMatches = Number(message.version) === EXTENSION_PROTOCOL_VERSION;
605
+ const validationLimit = Number(this.registry.urlValidationLimit) || DEFAULT_URL_VALIDATION_LIMIT;
606
+ if (
607
+ versionMatches
608
+ && url.length > 0
609
+ && url.length <= MAX_URL_VALIDATION_CHARS
610
+ && this.urlValidationCount < validationLimit
611
+ && typeof this.registry.validateBrowserUrl === 'function'
612
+ ) {
613
+ this.urlValidationCount += 1;
614
+ try {
615
+ const result = await this.registry.validateBrowserUrl(url, {
616
+ signal: this.urlValidationController.signal,
617
+ });
618
+ allowed = result?.allowed === true;
619
+ } catch {
620
+ allowed = false;
621
+ } finally {
622
+ this.urlValidationCount -= 1;
623
+ }
624
+ }
625
+
626
+ if (!this.isOpen()) return;
627
+ try {
628
+ this.ws.send(JSON.stringify({
629
+ type: EXTENSION_MESSAGE_TYPES.URL_VALIDATION_RESULT,
630
+ version: EXTENSION_PROTOCOL_VERSION,
631
+ id,
632
+ allowed,
633
+ }));
634
+ } catch {}
520
635
  }
521
636
 
522
637
  #closePending(error) {
638
+ this.closed = true;
639
+ if (!this.urlValidationController.signal.aborted) {
640
+ this.urlValidationController.abort(error);
641
+ }
523
642
  this.registry.unregisterConnection(this);
524
643
  if (this.heartbeatTimer) {
525
644
  clearInterval(this.heartbeatTimer);
@@ -527,6 +646,7 @@ class ExtensionBrowserConnection {
527
646
  }
528
647
  for (const pending of this.pending.values()) {
529
648
  clearTimeout(pending.timer);
649
+ pending.signal?.removeEventListener('abort', pending.onAbort);
530
650
  pending.reject(error);
531
651
  }
532
652
  this.pending.clear();
@@ -559,6 +679,7 @@ class ExtensionBrowserConnection {
559
679
  }
560
680
 
561
681
  module.exports = {
682
+ ExtensionBrowserConnection,
562
683
  BrowserExtensionRegistry,
563
684
  sha256,
564
685
  };