neoagent 2.3.1-beta.2 → 2.3.1-beta.3

package/README.md

@@ -22,6 +22,8 @@
 ```bash
 npm install -g neoagent
 neoagent install
+
+neoagent migrate
 ```
 
 ## Manage the Service

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "neoagent",
-  "version": "2.3.1-beta.2",
+  "version": "2.3.1-beta.3",
   "description": "Proactive personal AI agent with no limits",
   "license": "MIT",
   "main": "server/index.js",
@@ -80,6 +80,7 @@
     "socket.io": "^4.8.1",
     "telegraf": "^4.16.3",
     "telnyx": "^5.51.0",
+    "tesseract.js": "^7.0.0",
     "uuid": "^11.1.0",
     "ws": "^8.19.0"
   },

package/server/db/database.js

@@ -651,6 +651,40 @@ db.exec(`
   CREATE INDEX IF NOT EXISTS idx_recording_chunks_source ON recording_chunks(source_id, sequence_index);
   CREATE INDEX IF NOT EXISTS idx_recording_segments_session ON recording_transcript_segments(session_id, start_ms, created_at);
 
+  CREATE TABLE IF NOT EXISTS screen_history (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id INTEGER NOT NULL,
+    timestamp TEXT DEFAULT (datetime('now')),
+    app_name TEXT,
+    text_content TEXT,
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+  );
+
+  CREATE TABLE IF NOT EXISTS notification_history (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id INTEGER NOT NULL,
+    app_package TEXT,
+    title TEXT,
+    body TEXT,
+    timestamp TEXT DEFAULT (datetime('now')),
+    action_taken TEXT,
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+  );
+
+  CREATE TABLE IF NOT EXISTS geofences (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id INTEGER NOT NULL,
+    label TEXT,
+    latitude REAL NOT NULL,
+    longitude REAL NOT NULL,
+    radius_meters INTEGER NOT NULL,
+    trigger_action TEXT,
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_screen_history_user ON screen_history(user_id, timestamp DESC);
+  CREATE INDEX IF NOT EXISTS idx_notification_history_user ON notification_history(user_id, timestamp DESC);
+
   CREATE TABLE IF NOT EXISTS artifacts (
     id TEXT PRIMARY KEY,
     user_id INTEGER NOT NULL,
@@ -676,6 +710,29 @@ db.exec(`
 
 try {
   db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS screen_history_fts USING fts5(
+      text_content,
+      app_name,
+      timestamp UNINDEXED,
+      user_id UNINDEXED,
+      tokenize = 'porter unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS screen_history_fts_ai AFTER INSERT ON screen_history BEGIN
+      INSERT INTO screen_history_fts(rowid, text_content, app_name, timestamp, user_id)
+      VALUES (new.id, COALESCE(new.text_content, ''), COALESCE(new.app_name, ''), new.timestamp, new.user_id);
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS screen_history_fts_ad AFTER DELETE ON screen_history BEGIN
+      DELETE FROM screen_history_fts WHERE rowid = old.id;
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS screen_history_fts_au AFTER UPDATE ON screen_history BEGIN
+      DELETE FROM screen_history_fts WHERE rowid = old.id;
+      INSERT INTO screen_history_fts(rowid, text_content, app_name, timestamp, user_id)
+      VALUES (new.id, COALESCE(new.text_content, ''), COALESCE(new.app_name, ''), new.timestamp, new.user_id);
+    END;
+
     CREATE VIRTUAL TABLE IF NOT EXISTS conversation_history_fts USING fts5(
       content,
       role UNINDEXED,

package/server/http/routes/screenHistory.js

@@ -0,0 +1,46 @@
+'use strict';
+
+const express = require('express');
+const db = require('../../db/database');
+const { getErrorMessage } = require('../bootstrap_helpers');
+
+const router = express.Router();
+
+router.get('/search', (req, res) => {
+  const { q, limit = 50, offset = 0 } = req.query;
+
+  if (!req.user || !req.user.id) {
+    return res.status(401).json({ error: 'Unauthorized' });
+  }
+
+  try {
+    let results = [];
+    if (q) {
+      // Full text search
+      results = db.prepare(\`
+        SELECT s.id, s.timestamp, s.app_name, s.text_content
+        FROM screen_history_fts fts
+        JOIN screen_history s ON fts.rowid = s.id
+        WHERE screen_history_fts MATCH ? AND s.user_id = ?
+        ORDER BY s.timestamp DESC
+        LIMIT ? OFFSET ?
+      \`).all(q, req.user.id, Number(limit), Number(offset));
+    } else {
+      // Recent history
+      results = db.prepare(\`
+        SELECT id, timestamp, app_name, text_content
+        FROM screen_history
+        WHERE user_id = ?
+        ORDER BY timestamp DESC
+        LIMIT ? OFFSET ?
+      \`).all(req.user.id, Number(limit), Number(offset));
+    }
+
+    res.json({ results });
+  } catch (err) {
+    console.error('[ScreenHistory] Search error:', getErrorMessage(err));
+    res.status(500).json({ error: 'Failed to search screen history' });
+  }
+});
+
+module.exports = router;

package/server/http/routes/triggers.js

@@ -0,0 +1,81 @@
+'use strict';
+
+const express = require('express');
+const db = require('../../db/database');
+const { getErrorMessage } = require('../bootstrap_helpers');
+const { requireAuth } = require('../middleware/auth');
+
+const router = express.Router();
+
+router.use(requireAuth);
+
+router.post('/geofence', async (req, res) => {
+  const { label, latitude, longitude, radius_meters, action } = req.body;
+
+  try {
+    const userRow = db.prepare('SELECT id FROM users WHERE id = ?').get(req.user.id);
+    if (!userRow) return res.status(401).json({ error: 'Unauthorized' });
+
+    console.log(`[Triggers] Geofence entered: ${label} by user ${req.user.id}`);
+
+    // If an agentEngine is running, we can inject a prompt to process this context
+    const agentEngine = req.app.locals.agentEngine;
+    if (agentEngine) {
+      // Find active agent or use default
+      const defaultAgentId = db.prepare('SELECT id FROM agents WHERE user_id = ? ORDER BY is_default DESC LIMIT 1').get(req.user.id)?.id;
+      
+      if (defaultAgentId) {
+        // Fire and forget a trigger message to the agent
+        agentEngine.handleBackgroundTrigger(req.user.id, defaultAgentId, {
+          source: 'geofence',
+          label,
+          action: action || 'User entered a geofenced area. Check if there are any active reminders or tasks related to this location.'
+        }).catch(err => console.error('[Triggers] Agent evaluation failed:', err));
+      }
+    }
+
+    res.json({ success: true, message: 'Geofence trigger processed' });
+  } catch (err) {
+    console.error('[Triggers] Geofence error:', getErrorMessage(err));
+    res.status(500).json({ error: 'Failed to process geofence trigger' });
+  }
+});
+
+router.post('/notification', async (req, res) => {
+  const { app_package, title, body, action_taken } = req.body;
+
+  try {
+    const userRow = db.prepare('SELECT id FROM users WHERE id = ?').get(req.user.id);
+    if (!userRow) return res.status(401).json({ error: 'Unauthorized' });
+
+    console.log(`[Triggers] Notification received: ${app_package} - ${title}`);
+
+    db.prepare(\`
+      INSERT INTO notification_history (user_id, app_package, title, body, action_taken)
+      VALUES (?, ?, ?, ?, ?)
+    \`).run(req.user.id, app_package || 'unknown', title || '', body || '', action_taken || 'none');
+
+    // Notify agent engine to proactively evaluate the notification
+    const agentEngine = req.app.locals.agentEngine;
+    if (agentEngine) {
+      const defaultAgentId = db.prepare('SELECT id FROM agents WHERE user_id = ? ORDER BY is_default DESC LIMIT 1').get(req.user.id)?.id;
+      
+      if (defaultAgentId) {
+        agentEngine.handleBackgroundTrigger(req.user.id, defaultAgentId, {
+          source: 'notification',
+          app_package,
+          title,
+          body,
+          instruction: 'Evaluate this notification. If it is an important reminder, calendar event, or urgent message, inform the user or take appropriate action.'
+        }).catch(err => console.error('[Triggers] Agent evaluation failed:', err));
+      }
+    }
+
+    res.json({ success: true, message: 'Notification trigger processed and stored' });
+  } catch (err) {
+    console.error('[Triggers] Notification error:', getErrorMessage(err));
+    res.status(500).json({ error: 'Failed to process notification trigger' });
+  }
+});
+
+module.exports = router;

package/server/http/routes.js

@@ -26,7 +26,9 @@ const routeRegistry = [
   { basePath: '/api/desktop', modulePath: '../routes/desktop' },
   { basePath: '/api/recordings', modulePath: '../routes/recordings' },
   { basePath: '/api/voice-assistant', modulePath: '../routes/voice_assistant' },
-  { basePath: '/api/mobile/health', modulePath: '../routes/mobile-health' }
+  { basePath: '/api/mobile/health', modulePath: '../routes/mobile-health' },
+  { basePath: '/api/screen-history', modulePath: '../routes/screenHistory' },
+  { basePath: '/api/triggers', modulePath: '../routes/triggers' }
 ];
 
 function registerApiRoutes(app) {

package/server/public/assets/NOTICES

@@ -5535,6 +5535,19 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 --------------------------------------------------------------------------------
+flutter_background_service
+flutter_background_service_android
+flutter_background_service_ios
+flutter_background_service_platform_interface
+
+Copyright 2022 Eka Setiawan Saputra
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+--------------------------------------------------------------------------------
 flutter_lints
 flutter_markdown
 path_provider
@@ -7093,6 +7106,35 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 --------------------------------------------------------------------------------
+geolocator
+geolocator_android
+geolocator_apple
+geolocator_platform_interface
+geolocator_web
+geolocator_windows
+
+MIT License
+
+Copyright (c) 2018 Baseflow
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+--------------------------------------------------------------------------------
 glfw
 
 
@@ -24797,6 +24839,25 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 --------------------------------------------------------------------------------
+notification_listener_service
+
+Copyright (c) 2022 Iheb Briki
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+--------------------------------------------------------------------------------
 perfetto
 
                                  Apache License

package/server/public/flutter_bootstrap.js

@@ -37,6 +37,6 @@ _flutter.buildConfig = {"engineRevision":"59aa584fdf100e6c78c785d8a5b565d1de4b48
 
 _flutter.loader.load({
   serviceWorkerSettings: {
-    serviceWorkerVersion: "765364712" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
+    serviceWorkerVersion: "1375002290" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
   }
 });