neoagent 2.3.1-beta.85 → 2.3.1-beta.87

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/docs/capabilities.md +2 -0
  2. package/flutter_app/android/app/src/main/AndroidManifest.xml +14 -0
  3. package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +84 -0
  4. package/flutter_app/lib/main_chat.dart +156 -2
  5. package/flutter_app/lib/main_controller.dart +137 -10
  6. package/flutter_app/lib/main_models.dart +69 -0
  7. package/flutter_app/lib/main_operations.dart +248 -0
  8. package/flutter_app/lib/main_runtime.dart +11 -2
  9. package/flutter_app/lib/main_settings.dart +173 -176
  10. package/flutter_app/lib/main_shared.dart +78 -0
  11. package/flutter_app/lib/src/app_launch_bridge.dart +39 -10
  12. package/flutter_app/lib/src/backend_client.dart +28 -0
  13. package/package.json +1 -1
  14. package/server/guest-agent.android.package.json +13 -0
  15. package/server/guest-agent.browser.package.json +14 -0
  16. package/server/guest_agent.js +61 -44
  17. package/server/http/routes.js +1 -0
  18. package/server/public/.last_build_id +1 -1
  19. package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
  20. package/server/public/flutter_bootstrap.js +1 -1
  21. package/server/public/main.dart.js +69936 -69277
  22. package/server/routes/android.js +2 -11
  23. package/server/routes/browser.js +2 -2
  24. package/server/routes/memory.js +90 -0
  25. package/server/routes/social_video.js +62 -0
  26. package/server/services/ai/capabilityHealth.js +6 -14
  27. package/server/services/ai/systemPrompt.js +1 -0
  28. package/server/services/ai/toolResult.js +20 -0
  29. package/server/services/ai/tools.js +29 -0
  30. package/server/services/android/android_bootstrap_worker.js +2 -2
  31. package/server/services/android/controller.js +528 -132
  32. package/server/services/browser/controller.js +51 -68
  33. package/server/services/manager.js +15 -0
  34. package/server/services/memory/llm_transfer.js +217 -0
  35. package/server/services/runtime/backends/local-vm.js +16 -3
  36. package/server/services/runtime/guest_bootstrap.js +224 -56
  37. package/server/services/runtime/manager.js +53 -15
  38. package/server/services/runtime/qemu.js +149 -24
  39. package/server/services/runtime/settings.js +9 -14
  40. package/server/services/runtime/validation.js +10 -11
  41. package/server/services/social_video/adapters/base.js +26 -0
  42. package/server/services/social_video/adapters/index.js +27 -0
  43. package/server/services/social_video/adapters/instagram.js +17 -0
  44. package/server/services/social_video/adapters/tiktok.js +17 -0
  45. package/server/services/social_video/adapters/x.js +17 -0
  46. package/server/services/social_video/adapters/youtube.js +17 -0
  47. package/server/services/social_video/captions.js +187 -0
  48. package/server/services/social_video/frame.js +42 -0
  49. package/server/services/social_video/index.js +7 -0
  50. package/server/services/social_video/metadata.js +63 -0
  51. package/server/services/social_video/result.js +63 -0
  52. package/server/services/social_video/service.js +576 -0
  53. package/server/services/social_video/url.js +83 -0
  54. package/server/utils/deployment.js +4 -4
  55. package/server/guest-agent.package.json +0 -15
@@ -5,8 +5,6 @@ const fs = require('fs');
5
5
  const os = require('os');
6
6
  const path = require('path');
7
7
  const { CLIExecutor } = require('./services/cli/executor');
8
- const { BrowserController } = require('./services/browser/controller');
9
- const { AndroidController } = require('./services/android/controller');
10
8
  const { RUNTIME_HOME } = require('../runtime/paths');
11
9
 
12
10
  const PORT = Number(process.env.NEOAGENT_GUEST_AGENT_PORT || 8421);
@@ -23,6 +21,9 @@ function resolveGuestToken() {
23
21
  }
24
22
 
25
23
  const AUTH_TOKEN = resolveGuestToken();
24
+ const GUEST_PROFILE = String(process.env.NEOAGENT_GUEST_PROFILE || 'browser_cli').trim() === 'android'
25
+ ? 'android'
26
+ : 'browser_cli';
26
27
  const FILE_ROOT = path.join(RUNTIME_HOME, 'guest-agent-files');
27
28
  const MAX_APK_STREAM_BYTES = Number(process.env.NEOAGENT_GUEST_MAX_APK_STREAM_BYTES || 512 * 1024 * 1024);
28
29
 
@@ -32,8 +33,12 @@ const app = express();
32
33
  app.use(express.json({ limit: '100mb' }));
33
34
 
34
35
  const cliExecutor = new CLIExecutor();
35
- const browserController = new BrowserController({ runtimeBackend: 'vm' });
36
- const androidController = new AndroidController({ runtimeBackend: 'vm' });
36
+ const browserController = GUEST_PROFILE === 'browser_cli'
37
+ ? new (require('./services/browser/controller').BrowserController)({ runtimeBackend: 'vm' })
38
+ : null;
39
+ const androidController = GUEST_PROFILE === 'android'
40
+ ? new (require('./services/android/controller').AndroidController)({ runtimeBackend: 'vm' })
41
+ : null;
37
42
 
38
43
  const ALLOWED_READABLE_ROOTS = [
39
44
  FILE_ROOT,
@@ -85,10 +90,10 @@ async function handle(res, work) {
85
90
  app.use(requireToken);
86
91
 
87
92
  app.get('/health', (_req, res) => {
88
- console.log('[Guest Agent] Health check requested');
89
93
  res.json({
90
94
  status: 'ok',
91
95
  runtime: 'guest-agent',
96
+ profile: GUEST_PROFILE,
92
97
  platform: process.platform,
93
98
  arch: process.arch,
94
99
  });
@@ -151,45 +156,56 @@ app.post('/files/read', async (req, res) => {
151
156
  });
152
157
 
153
158
  app.get('/browser/status', async (_req, res) => {
154
- await handle(res, async () => ({
155
- launched: await Promise.resolve(browserController.isLaunched()),
156
- pages: await Promise.resolve(browserController.getPageCount()),
157
- headless: browserController.headless,
158
- pageInfo: await browserController.getPageInfo(),
159
- }));
159
+ await handle(res, async () => {
160
+ const controller = requireCapability(browserController, 'browser');
161
+ return {
162
+ launched: await Promise.resolve(controller.isLaunched()),
163
+ pages: await Promise.resolve(controller.getPageCount()),
164
+ headless: controller.headless,
165
+ pageInfo: await controller.getPageInfo(),
166
+ };
167
+ });
160
168
  });
161
- app.post('/browser/launch', async (req, res) => handle(res, () => browserController.launch(req.body || {})));
162
- app.post('/browser/navigate', async (req, res) => handle(res, () => browserController.navigate(req.body?.url, req.body || {})));
163
- app.post('/browser/screenshot', async (req, res) => handle(res, () => browserController.screenshot(req.body || {})));
164
- app.post('/browser/click', async (req, res) => handle(res, () => browserController.click(req.body?.selector, req.body?.text, req.body?.screenshot !== false)));
165
- app.post('/browser/click-point', async (req, res) => handle(res, () => browserController.clickPoint(req.body?.x, req.body?.y, req.body?.screenshot !== false)));
166
- app.post('/browser/fill', async (req, res) => handle(res, () => browserController.type(req.body?.selector, String(req.body?.value ?? req.body?.text ?? ''), req.body || {})));
167
- app.post('/browser/type-text', async (req, res) => handle(res, () => browserController.typeText(String(req.body?.text || ''), req.body || {})));
168
- app.post('/browser/press-key', async (req, res) => handle(res, () => browserController.pressKey(req.body?.key, req.body?.screenshot !== false)));
169
- app.post('/browser/scroll', async (req, res) => handle(res, () => browserController.scroll(req.body?.deltaX ?? 0, req.body?.deltaY ?? 0, req.body?.screenshot !== false)));
170
- app.post('/browser/extract', async (req, res) => handle(res, () => browserController.extractContent(req.body || {})));
171
- app.post('/browser/execute', async (req, res) => handle(res, () => browserController.executeJS(req.body?.code)));
172
- app.post('/browser/close', async (_req, res) => handle(res, () => browserController.closeBrowser().then(() => ({ success: true }))));
169
+ function requireCapability(controller, name) {
170
+ if (!controller) {
171
+ throw new Error(`${name} runtime is unavailable in this guest profile.`);
172
+ }
173
+ return controller;
174
+ }
175
+
176
+ app.post('/browser/launch', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').launch(req.body || {})));
177
+ app.post('/browser/navigate', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').navigate(req.body?.url, req.body || {})));
178
+ app.post('/browser/screenshot', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').screenshot(req.body || {})));
179
+ app.post('/browser/click', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').click(req.body?.selector, req.body?.text, req.body?.screenshot !== false)));
180
+ app.post('/browser/click-point', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').clickPoint(req.body?.x, req.body?.y, req.body?.screenshot !== false)));
181
+ app.post('/browser/fill', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').type(req.body?.selector, String(req.body?.value ?? req.body?.text ?? ''), req.body || {})));
182
+ app.post('/browser/type-text', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').typeText(String(req.body?.text || ''), req.body || {})));
183
+ app.post('/browser/press-key', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').pressKey(req.body?.key, req.body?.screenshot !== false)));
184
+ app.post('/browser/scroll', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').scroll(req.body?.deltaX ?? 0, req.body?.deltaY ?? 0, req.body?.screenshot !== false)));
185
+ app.post('/browser/extract', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').extractContent(req.body || {})));
186
+ app.post('/browser/execute', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').executeJS(req.body?.code)));
187
+ app.post('/browser/close', async (_req, res) => handle(res, () => requireCapability(browserController, 'browser').closeBrowser().then(() => ({ success: true }))));
173
188
 
174
- app.get('/android/status', async (_req, res) => handle(res, () => androidController.getStatus()));
175
- app.post('/android/start', async (req, res) => handle(res, () => androidController.requestStartEmulator(req.body || {})));
176
- app.post('/android/stop', async (_req, res) => handle(res, () => androidController.stopEmulator()));
177
- app.get('/android/devices', async (_req, res) => handle(res, async () => ({ devices: await androidController.listDevices() })));
178
- app.post('/android/screenshot', async (req, res) => handle(res, () => androidController.screenshot(req.body || {})));
179
- app.post('/android/observe', async (req, res) => handle(res, () => androidController.observe(req.body || {})));
180
- app.post('/android/ui-dump', async (req, res) => handle(res, () => androidController.dumpUi(req.body || {})));
181
- app.get('/android/apps', async (req, res) => handle(res, () => androidController.listApps({ includeSystem: req.query.includeSystem === 'true' })));
182
- app.post('/android/open-app', async (req, res) => handle(res, () => androidController.openApp(req.body || {})));
183
- app.post('/android/open-intent', async (req, res) => handle(res, () => androidController.openIntent(req.body || {})));
184
- app.post('/android/tap', async (req, res) => handle(res, () => androidController.tap(req.body || {})));
185
- app.post('/android/long-press', async (req, res) => handle(res, () => androidController.longPress(req.body || {})));
186
- app.post('/android/type', async (req, res) => handle(res, () => androidController.type(req.body || {})));
187
- app.post('/android/swipe', async (req, res) => handle(res, () => androidController.swipe(req.body || {})));
188
- app.post('/android/press-key', async (req, res) => handle(res, () => androidController.pressKey(req.body || {})));
189
- app.post('/android/wait-for', async (req, res) => handle(res, () => androidController.waitFor(req.body || {})));
190
- app.post('/android/shell', async (req, res) => handle(res, () => androidController.shell(req.body || {})));
189
+ app.get('/android/status', async (_req, res) => handle(res, () => requireCapability(androidController, 'android').getStatus()));
190
+ app.post('/android/start', async (req, res) => handle(res, () => requireCapability(androidController, 'android').requestStartEmulator(req.body || {})));
191
+ app.post('/android/stop', async (_req, res) => handle(res, () => requireCapability(androidController, 'android').stopEmulator()));
192
+ app.get('/android/devices', async (_req, res) => handle(res, async () => ({ devices: await requireCapability(androidController, 'android').listDevices() })));
193
+ app.post('/android/screenshot', async (req, res) => handle(res, () => requireCapability(androidController, 'android').screenshot(req.body || {})));
194
+ app.post('/android/observe', async (req, res) => handle(res, () => requireCapability(androidController, 'android').observe(req.body || {})));
195
+ app.post('/android/ui-dump', async (req, res) => handle(res, () => requireCapability(androidController, 'android').dumpUi(req.body || {})));
196
+ app.get('/android/apps', async (req, res) => handle(res, () => requireCapability(androidController, 'android').listApps({ includeSystem: req.query.includeSystem === 'true' })));
197
+ app.post('/android/open-app', async (req, res) => handle(res, () => requireCapability(androidController, 'android').openApp(req.body || {})));
198
+ app.post('/android/open-intent', async (req, res) => handle(res, () => requireCapability(androidController, 'android').openIntent(req.body || {})));
199
+ app.post('/android/tap', async (req, res) => handle(res, () => requireCapability(androidController, 'android').tap(req.body || {})));
200
+ app.post('/android/long-press', async (req, res) => handle(res, () => requireCapability(androidController, 'android').longPress(req.body || {})));
201
+ app.post('/android/type', async (req, res) => handle(res, () => requireCapability(androidController, 'android').type(req.body || {})));
202
+ app.post('/android/swipe', async (req, res) => handle(res, () => requireCapability(androidController, 'android').swipe(req.body || {})));
203
+ app.post('/android/press-key', async (req, res) => handle(res, () => requireCapability(androidController, 'android').pressKey(req.body || {})));
204
+ app.post('/android/wait-for', async (req, res) => handle(res, () => requireCapability(androidController, 'android').waitFor(req.body || {})));
205
+ app.post('/android/shell', async (req, res) => handle(res, () => requireCapability(androidController, 'android').shell(req.body || {})));
191
206
  app.post('/android/install-apk', async (req, res) => {
192
207
  await handle(res, async () => {
208
+ requireCapability(androidController, 'android');
193
209
  const filename = String(req.body?.filename || 'upload.apk').trim() || 'upload.apk';
194
210
  const contentBase64 = String(req.body?.contentBase64 || '').trim();
195
211
  if (!contentBase64) {
@@ -255,6 +271,7 @@ app.post('/android/install-apk-stream', async (req, res) => {
255
271
  return;
256
272
  }
257
273
  try {
274
+ requireCapability(androidController, 'android');
258
275
  const result = await androidController.installApk({ apkPath: tempPath });
259
276
  finished = true;
260
277
  await cleanup();
@@ -267,18 +284,18 @@ app.post('/android/install-apk-stream', async (req, res) => {
267
284
  req.pipe(output);
268
285
  });
269
286
 
270
- const server = app.listen(PORT, () => {
271
- console.log(`NeoAgent guest agent listening on http://127.0.0.1:${PORT}`);
287
+ const server = app.listen(PORT, '0.0.0.0', () => {
288
+ console.log(`NeoAgent guest agent listening on http://0.0.0.0:${PORT}`);
272
289
  });
273
290
 
274
291
  async function shutdown() {
275
292
  try {
276
- await browserController.closeBrowser();
293
+ await browserController?.closeBrowser?.();
277
294
  } catch (err) {
278
295
  console.warn('[GuestAgent] Failed to close browser:', err?.message);
279
296
  }
280
297
  try {
281
- await androidController.close();
298
+ await androidController?.close?.();
282
299
  } catch (err) {
283
300
  console.warn('[GuestAgent] Failed to close android controller:', err?.message);
284
301
  }
@@ -26,6 +26,7 @@ const routeRegistry = [
26
26
  { basePath: '/api/android', modulePath: '../routes/android' },
27
27
  { basePath: '/api/desktop', modulePath: '../routes/desktop' },
28
28
  { basePath: '/api/recordings', modulePath: '../routes/recordings' },
29
+ { basePath: '/api/social-video', modulePath: '../routes/social_video' },
29
30
  { basePath: '/api/voice-assistant', modulePath: '../routes/voice_assistant' },
30
31
  { basePath: '/api/wearable', modulePath: '../routes/wearable' },
31
32
  { basePath: '/api/mobile/health', modulePath: '../routes/mobile-health' },
@@ -1 +1 @@
1
- 76e31ddbfeee7f5921472f8ab23aa471
1
+ 1009faa3470f07884870d6f46bd7cd4d
@@ -37,6 +37,6 @@ _flutter.buildConfig = {"engineRevision":"42d3d75a56efe1a2e9902f52dc8006099c45d9
37
37
 
38
38
  _flutter.loader.load({
39
39
  serviceWorkerSettings: {
40
- serviceWorkerVersion: "3623988917" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
40
+ serviceWorkerVersion: "1661842067" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
41
41
  }
42
42
  });