neoagent 2.3.1-beta.85 → 2.3.1-beta.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/capabilities.md +2 -0
- package/flutter_app/android/app/src/main/AndroidManifest.xml +14 -0
- package/flutter_app/android/app/src/main/kotlin/com/neoagent/flutter_app/MainActivity.kt +84 -0
- package/flutter_app/lib/main_chat.dart +156 -2
- package/flutter_app/lib/main_controller.dart +137 -10
- package/flutter_app/lib/main_models.dart +69 -0
- package/flutter_app/lib/main_operations.dart +248 -0
- package/flutter_app/lib/main_runtime.dart +11 -2
- package/flutter_app/lib/main_settings.dart +173 -176
- package/flutter_app/lib/main_shared.dart +78 -0
- package/flutter_app/lib/src/app_launch_bridge.dart +39 -10
- package/flutter_app/lib/src/backend_client.dart +28 -0
- package/package.json +1 -1
- package/server/guest-agent.android.package.json +13 -0
- package/server/guest-agent.browser.package.json +14 -0
- package/server/guest_agent.js +61 -44
- package/server/http/routes.js +1 -0
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/main.dart.js +69936 -69277
- package/server/routes/android.js +2 -11
- package/server/routes/browser.js +2 -2
- package/server/routes/memory.js +90 -0
- package/server/routes/social_video.js +62 -0
- package/server/services/ai/capabilityHealth.js +6 -14
- package/server/services/ai/systemPrompt.js +1 -0
- package/server/services/ai/toolResult.js +20 -0
- package/server/services/ai/tools.js +29 -0
- package/server/services/android/android_bootstrap_worker.js +2 -2
- package/server/services/android/controller.js +528 -132
- package/server/services/browser/controller.js +51 -68
- package/server/services/manager.js +15 -0
- package/server/services/memory/llm_transfer.js +217 -0
- package/server/services/runtime/backends/local-vm.js +16 -3
- package/server/services/runtime/guest_bootstrap.js +224 -56
- package/server/services/runtime/manager.js +53 -15
- package/server/services/runtime/qemu.js +149 -24
- package/server/services/runtime/settings.js +9 -14
- package/server/services/runtime/validation.js +10 -11
- package/server/services/social_video/adapters/base.js +26 -0
- package/server/services/social_video/adapters/index.js +27 -0
- package/server/services/social_video/adapters/instagram.js +17 -0
- package/server/services/social_video/adapters/tiktok.js +17 -0
- package/server/services/social_video/adapters/x.js +17 -0
- package/server/services/social_video/adapters/youtube.js +17 -0
- package/server/services/social_video/captions.js +187 -0
- package/server/services/social_video/frame.js +42 -0
- package/server/services/social_video/index.js +7 -0
- package/server/services/social_video/metadata.js +63 -0
- package/server/services/social_video/result.js +63 -0
- package/server/services/social_video/service.js +576 -0
- package/server/services/social_video/url.js +83 -0
- package/server/utils/deployment.js +4 -4
- package/server/guest-agent.package.json +0 -15
package/server/guest_agent.js
CHANGED
|
@@ -5,8 +5,6 @@ const fs = require('fs');
|
|
|
5
5
|
const os = require('os');
|
|
6
6
|
const path = require('path');
|
|
7
7
|
const { CLIExecutor } = require('./services/cli/executor');
|
|
8
|
-
const { BrowserController } = require('./services/browser/controller');
|
|
9
|
-
const { AndroidController } = require('./services/android/controller');
|
|
10
8
|
const { RUNTIME_HOME } = require('../runtime/paths');
|
|
11
9
|
|
|
12
10
|
const PORT = Number(process.env.NEOAGENT_GUEST_AGENT_PORT || 8421);
|
|
@@ -23,6 +21,9 @@ function resolveGuestToken() {
|
|
|
23
21
|
}
|
|
24
22
|
|
|
25
23
|
const AUTH_TOKEN = resolveGuestToken();
|
|
24
|
+
const GUEST_PROFILE = String(process.env.NEOAGENT_GUEST_PROFILE || 'browser_cli').trim() === 'android'
|
|
25
|
+
? 'android'
|
|
26
|
+
: 'browser_cli';
|
|
26
27
|
const FILE_ROOT = path.join(RUNTIME_HOME, 'guest-agent-files');
|
|
27
28
|
const MAX_APK_STREAM_BYTES = Number(process.env.NEOAGENT_GUEST_MAX_APK_STREAM_BYTES || 512 * 1024 * 1024);
|
|
28
29
|
|
|
@@ -32,8 +33,12 @@ const app = express();
|
|
|
32
33
|
app.use(express.json({ limit: '100mb' }));
|
|
33
34
|
|
|
34
35
|
const cliExecutor = new CLIExecutor();
|
|
35
|
-
const browserController =
|
|
36
|
-
|
|
36
|
+
const browserController = GUEST_PROFILE === 'browser_cli'
|
|
37
|
+
? new (require('./services/browser/controller').BrowserController)({ runtimeBackend: 'vm' })
|
|
38
|
+
: null;
|
|
39
|
+
const androidController = GUEST_PROFILE === 'android'
|
|
40
|
+
? new (require('./services/android/controller').AndroidController)({ runtimeBackend: 'vm' })
|
|
41
|
+
: null;
|
|
37
42
|
|
|
38
43
|
const ALLOWED_READABLE_ROOTS = [
|
|
39
44
|
FILE_ROOT,
|
|
@@ -85,10 +90,10 @@ async function handle(res, work) {
|
|
|
85
90
|
app.use(requireToken);
|
|
86
91
|
|
|
87
92
|
app.get('/health', (_req, res) => {
|
|
88
|
-
console.log('[Guest Agent] Health check requested');
|
|
89
93
|
res.json({
|
|
90
94
|
status: 'ok',
|
|
91
95
|
runtime: 'guest-agent',
|
|
96
|
+
profile: GUEST_PROFILE,
|
|
92
97
|
platform: process.platform,
|
|
93
98
|
arch: process.arch,
|
|
94
99
|
});
|
|
@@ -151,45 +156,56 @@ app.post('/files/read', async (req, res) => {
|
|
|
151
156
|
});
|
|
152
157
|
|
|
153
158
|
app.get('/browser/status', async (_req, res) => {
|
|
154
|
-
await handle(res, async () =>
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
159
|
+
await handle(res, async () => {
|
|
160
|
+
const controller = requireCapability(browserController, 'browser');
|
|
161
|
+
return {
|
|
162
|
+
launched: await Promise.resolve(controller.isLaunched()),
|
|
163
|
+
pages: await Promise.resolve(controller.getPageCount()),
|
|
164
|
+
headless: controller.headless,
|
|
165
|
+
pageInfo: await controller.getPageInfo(),
|
|
166
|
+
};
|
|
167
|
+
});
|
|
160
168
|
});
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
app.post('/browser/
|
|
169
|
-
app.post('/browser/
|
|
170
|
-
app.post('/browser/
|
|
171
|
-
app.post('/browser/
|
|
172
|
-
app.post('/browser/
|
|
169
|
+
function requireCapability(controller, name) {
|
|
170
|
+
if (!controller) {
|
|
171
|
+
throw new Error(`${name} runtime is unavailable in this guest profile.`);
|
|
172
|
+
}
|
|
173
|
+
return controller;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
app.post('/browser/launch', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').launch(req.body || {})));
|
|
177
|
+
app.post('/browser/navigate', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').navigate(req.body?.url, req.body || {})));
|
|
178
|
+
app.post('/browser/screenshot', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').screenshot(req.body || {})));
|
|
179
|
+
app.post('/browser/click', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').click(req.body?.selector, req.body?.text, req.body?.screenshot !== false)));
|
|
180
|
+
app.post('/browser/click-point', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').clickPoint(req.body?.x, req.body?.y, req.body?.screenshot !== false)));
|
|
181
|
+
app.post('/browser/fill', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').type(req.body?.selector, String(req.body?.value ?? req.body?.text ?? ''), req.body || {})));
|
|
182
|
+
app.post('/browser/type-text', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').typeText(String(req.body?.text || ''), req.body || {})));
|
|
183
|
+
app.post('/browser/press-key', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').pressKey(req.body?.key, req.body?.screenshot !== false)));
|
|
184
|
+
app.post('/browser/scroll', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').scroll(req.body?.deltaX ?? 0, req.body?.deltaY ?? 0, req.body?.screenshot !== false)));
|
|
185
|
+
app.post('/browser/extract', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').extractContent(req.body || {})));
|
|
186
|
+
app.post('/browser/execute', async (req, res) => handle(res, () => requireCapability(browserController, 'browser').executeJS(req.body?.code)));
|
|
187
|
+
app.post('/browser/close', async (_req, res) => handle(res, () => requireCapability(browserController, 'browser').closeBrowser().then(() => ({ success: true }))));
|
|
173
188
|
|
|
174
|
-
app.get('/android/status', async (_req, res) => handle(res, () => androidController.getStatus()));
|
|
175
|
-
app.post('/android/start', async (req, res) => handle(res, () => androidController.requestStartEmulator(req.body || {})));
|
|
176
|
-
app.post('/android/stop', async (_req, res) => handle(res, () => androidController.stopEmulator()));
|
|
177
|
-
app.get('/android/devices', async (_req, res) => handle(res, async () => ({ devices: await androidController.listDevices() })));
|
|
178
|
-
app.post('/android/screenshot', async (req, res) => handle(res, () => androidController.screenshot(req.body || {})));
|
|
179
|
-
app.post('/android/observe', async (req, res) => handle(res, () => androidController.observe(req.body || {})));
|
|
180
|
-
app.post('/android/ui-dump', async (req, res) => handle(res, () => androidController.dumpUi(req.body || {})));
|
|
181
|
-
app.get('/android/apps', async (req, res) => handle(res, () => androidController.listApps({ includeSystem: req.query.includeSystem === 'true' })));
|
|
182
|
-
app.post('/android/open-app', async (req, res) => handle(res, () => androidController.openApp(req.body || {})));
|
|
183
|
-
app.post('/android/open-intent', async (req, res) => handle(res, () => androidController.openIntent(req.body || {})));
|
|
184
|
-
app.post('/android/tap', async (req, res) => handle(res, () => androidController.tap(req.body || {})));
|
|
185
|
-
app.post('/android/long-press', async (req, res) => handle(res, () => androidController.longPress(req.body || {})));
|
|
186
|
-
app.post('/android/type', async (req, res) => handle(res, () => androidController.type(req.body || {})));
|
|
187
|
-
app.post('/android/swipe', async (req, res) => handle(res, () => androidController.swipe(req.body || {})));
|
|
188
|
-
app.post('/android/press-key', async (req, res) => handle(res, () => androidController.pressKey(req.body || {})));
|
|
189
|
-
app.post('/android/wait-for', async (req, res) => handle(res, () => androidController.waitFor(req.body || {})));
|
|
190
|
-
app.post('/android/shell', async (req, res) => handle(res, () => androidController.shell(req.body || {})));
|
|
189
|
+
app.get('/android/status', async (_req, res) => handle(res, () => requireCapability(androidController, 'android').getStatus()));
|
|
190
|
+
app.post('/android/start', async (req, res) => handle(res, () => requireCapability(androidController, 'android').requestStartEmulator(req.body || {})));
|
|
191
|
+
app.post('/android/stop', async (_req, res) => handle(res, () => requireCapability(androidController, 'android').stopEmulator()));
|
|
192
|
+
app.get('/android/devices', async (_req, res) => handle(res, async () => ({ devices: await requireCapability(androidController, 'android').listDevices() })));
|
|
193
|
+
app.post('/android/screenshot', async (req, res) => handle(res, () => requireCapability(androidController, 'android').screenshot(req.body || {})));
|
|
194
|
+
app.post('/android/observe', async (req, res) => handle(res, () => requireCapability(androidController, 'android').observe(req.body || {})));
|
|
195
|
+
app.post('/android/ui-dump', async (req, res) => handle(res, () => requireCapability(androidController, 'android').dumpUi(req.body || {})));
|
|
196
|
+
app.get('/android/apps', async (req, res) => handle(res, () => requireCapability(androidController, 'android').listApps({ includeSystem: req.query.includeSystem === 'true' })));
|
|
197
|
+
app.post('/android/open-app', async (req, res) => handle(res, () => requireCapability(androidController, 'android').openApp(req.body || {})));
|
|
198
|
+
app.post('/android/open-intent', async (req, res) => handle(res, () => requireCapability(androidController, 'android').openIntent(req.body || {})));
|
|
199
|
+
app.post('/android/tap', async (req, res) => handle(res, () => requireCapability(androidController, 'android').tap(req.body || {})));
|
|
200
|
+
app.post('/android/long-press', async (req, res) => handle(res, () => requireCapability(androidController, 'android').longPress(req.body || {})));
|
|
201
|
+
app.post('/android/type', async (req, res) => handle(res, () => requireCapability(androidController, 'android').type(req.body || {})));
|
|
202
|
+
app.post('/android/swipe', async (req, res) => handle(res, () => requireCapability(androidController, 'android').swipe(req.body || {})));
|
|
203
|
+
app.post('/android/press-key', async (req, res) => handle(res, () => requireCapability(androidController, 'android').pressKey(req.body || {})));
|
|
204
|
+
app.post('/android/wait-for', async (req, res) => handle(res, () => requireCapability(androidController, 'android').waitFor(req.body || {})));
|
|
205
|
+
app.post('/android/shell', async (req, res) => handle(res, () => requireCapability(androidController, 'android').shell(req.body || {})));
|
|
191
206
|
app.post('/android/install-apk', async (req, res) => {
|
|
192
207
|
await handle(res, async () => {
|
|
208
|
+
requireCapability(androidController, 'android');
|
|
193
209
|
const filename = String(req.body?.filename || 'upload.apk').trim() || 'upload.apk';
|
|
194
210
|
const contentBase64 = String(req.body?.contentBase64 || '').trim();
|
|
195
211
|
if (!contentBase64) {
|
|
@@ -255,6 +271,7 @@ app.post('/android/install-apk-stream', async (req, res) => {
|
|
|
255
271
|
return;
|
|
256
272
|
}
|
|
257
273
|
try {
|
|
274
|
+
requireCapability(androidController, 'android');
|
|
258
275
|
const result = await androidController.installApk({ apkPath: tempPath });
|
|
259
276
|
finished = true;
|
|
260
277
|
await cleanup();
|
|
@@ -267,18 +284,18 @@ app.post('/android/install-apk-stream', async (req, res) => {
|
|
|
267
284
|
req.pipe(output);
|
|
268
285
|
});
|
|
269
286
|
|
|
270
|
-
const server = app.listen(PORT, () => {
|
|
271
|
-
console.log(`NeoAgent guest agent listening on http://
|
|
287
|
+
const server = app.listen(PORT, '0.0.0.0', () => {
|
|
288
|
+
console.log(`NeoAgent guest agent listening on http://0.0.0.0:${PORT}`);
|
|
272
289
|
});
|
|
273
290
|
|
|
274
291
|
async function shutdown() {
|
|
275
292
|
try {
|
|
276
|
-
await browserController
|
|
293
|
+
await browserController?.closeBrowser?.();
|
|
277
294
|
} catch (err) {
|
|
278
295
|
console.warn('[GuestAgent] Failed to close browser:', err?.message);
|
|
279
296
|
}
|
|
280
297
|
try {
|
|
281
|
-
await androidController
|
|
298
|
+
await androidController?.close?.();
|
|
282
299
|
} catch (err) {
|
|
283
300
|
console.warn('[GuestAgent] Failed to close android controller:', err?.message);
|
|
284
301
|
}
|
package/server/http/routes.js
CHANGED
|
@@ -26,6 +26,7 @@ const routeRegistry = [
|
|
|
26
26
|
{ basePath: '/api/android', modulePath: '../routes/android' },
|
|
27
27
|
{ basePath: '/api/desktop', modulePath: '../routes/desktop' },
|
|
28
28
|
{ basePath: '/api/recordings', modulePath: '../routes/recordings' },
|
|
29
|
+
{ basePath: '/api/social-video', modulePath: '../routes/social_video' },
|
|
29
30
|
{ basePath: '/api/voice-assistant', modulePath: '../routes/voice_assistant' },
|
|
30
31
|
{ basePath: '/api/wearable', modulePath: '../routes/wearable' },
|
|
31
32
|
{ basePath: '/api/mobile/health', modulePath: '../routes/mobile-health' },
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
1009faa3470f07884870d6f46bd7cd4d
|
|
Binary file
|
|
@@ -37,6 +37,6 @@ _flutter.buildConfig = {"engineRevision":"42d3d75a56efe1a2e9902f52dc8006099c45d9
|
|
|
37
37
|
|
|
38
38
|
_flutter.loader.load({
|
|
39
39
|
serviceWorkerSettings: {
|
|
40
|
-
serviceWorkerVersion: "
|
|
40
|
+
serviceWorkerVersion: "1661842067" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
|
|
41
41
|
}
|
|
42
42
|
});
|