neoagent 2.3.1-beta.0 → 2.3.1-beta.100

package/server/db/database.js

@@ -1,4 +1,5 @@
 const Database = require('better-sqlite3');
+const fs = require('fs');
 const path = require('path');
 const { randomUUID } = require('crypto');
 const { DATA_DIR, ensureRuntimeDirs } = require('../../runtime/paths');
@@ -9,11 +10,85 @@ const {
 ensureRuntimeDirs();
 
 const DB_PATH = path.join(DATA_DIR, 'neoagent.db');
-const db = new Database(DB_PATH);
 
-db.pragma('journal_mode = WAL');
-db.pragma('foreign_keys = ON');
-db.pragma('busy_timeout = 5000');
+function removeWalSidecars(dbPath) {
+  for (const suffix of ['-wal', '-shm']) {
+    try {
+      fs.rmSync(`${dbPath}${suffix}`, { force: true });
+    } catch {}
+  }
+}
+
+function initializeDatabase(db, dbPath) {
+  try {
+    db.pragma('journal_mode = WAL');
+  } catch (error) {
+    console.warn(
+      `[Database] Failed to enable WAL for ${dbPath}: ${error.message}. ` +
+      'Retrying after clearing WAL sidecar files.',
+    );
+    try {
+      db.close();
+    } catch {}
+
+    removeWalSidecars(dbPath);
+
+    db = new Database(dbPath);
+    try {
+      db.pragma('journal_mode = WAL');
+    } catch (retryError) {
+      console.warn(
+        `[Database] WAL is unavailable for ${dbPath}: ${retryError.message}. ` +
+        'Falling back to DELETE journal mode.',
+      );
+      db.pragma('journal_mode = DELETE');
+    }
+  }
+
+  db.pragma('foreign_keys = ON');
+  db.pragma('busy_timeout = 5000');
+  return db;
+}
+
+function sleepSync(ms) {
+  if (ms <= 0) return;
+  const buffer = new SharedArrayBuffer(4);
+  const view = new Int32Array(buffer);
+  Atomics.wait(view, 0, 0, ms);
+}
+
+function isSqliteBusyError(error) {
+  return Boolean(
+    error &&
+    (
+      error.code === 'SQLITE_BUSY' ||
+      error.code === 'SQLITE_LOCKED' ||
+      /database is locked|database table is locked/i.test(error.message || '')
+    ),
+  );
+}
+
+function runWithBusyRetry(action, { attempts = 5, delayMs = 50, label = 'SQLite operation' } = {}) {
+  let lastError = null;
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    try {
+      return action();
+    } catch (error) {
+      lastError = error;
+      if (!isSqliteBusyError(error) || attempt === attempts) {
+        throw error;
+      }
+      console.warn(
+        `[Database] ${label} hit a busy lock on attempt ${attempt}/${attempts}; retrying in ${delayMs}ms.`,
+      );
+      sleepSync(delayMs);
+    }
+  }
+  throw lastError;
+}
+
+let db = new Database(DB_PATH);
+db = initializeDatabase(db, DB_PATH);
 
 db.exec(`
   CREATE TABLE IF NOT EXISTS users (
@@ -24,7 +99,8 @@ db.exec(`
     password TEXT NOT NULL,
     password_login_enabled INTEGER DEFAULT 1,
     created_at TEXT DEFAULT (datetime('now')),
-    last_login TEXT
+    last_login TEXT,
+    has_completed_onboarding INTEGER DEFAULT 0
   );
 
   CREATE TABLE IF NOT EXISTS agents (
@@ -272,6 +348,19 @@ db.exec(`
     FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL
   );
 
+  CREATE TABLE IF NOT EXISTS integration_provider_configs (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id INTEGER NOT NULL,
+    agent_id TEXT,
+    provider_key TEXT NOT NULL,
+    config_json TEXT DEFAULT '{}',
+    created_at TEXT DEFAULT (datetime('now')),
+    updated_at TEXT DEFAULT (datetime('now')),
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
+    FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL,
+    UNIQUE(user_id, agent_id, provider_key)
+  );
+
   CREATE TABLE IF NOT EXISTS browser_extension_pairing_requests (
     id TEXT PRIMARY KEY,
     user_id INTEGER,
@@ -419,6 +508,7 @@ db.exec(`
   CREATE INDEX IF NOT EXISTS idx_desktop_companion_devices_user ON desktop_companion_devices(user_id, status, created_at DESC);
   CREATE INDEX IF NOT EXISTS idx_messages_user ON messages(user_id, created_at DESC);
   CREATE INDEX IF NOT EXISTS idx_messages_platform ON messages(platform, platform_chat_id);
+  CREATE INDEX IF NOT EXISTS idx_messages_dedup ON messages(user_id, platform, platform_msg_id) WHERE platform_msg_id IS NOT NULL;
   CREATE INDEX IF NOT EXISTS idx_conv_messages ON conversation_messages(conversation_id, created_at);
   CREATE INDEX IF NOT EXISTS idx_conversations_user ON conversations(user_id, updated_at DESC);
   CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_user ON scheduled_tasks(user_id);
@@ -428,6 +518,9 @@ db.exec(`
     user_id INTEGER NOT NULL,
     agent_id TEXT,
     name TEXT NOT NULL,
+    widget_kind TEXT DEFAULT 'custom',
+    system_key TEXT,
+    is_system INTEGER DEFAULT 0,
     template TEXT NOT NULL,
     layout_variant TEXT NOT NULL,
     definition_json TEXT DEFAULT '{}',
@@ -479,6 +572,13 @@ db.exec(`
     user_id INTEGER NOT NULL,
     agent_id TEXT,
     category TEXT DEFAULT 'episodic',
+    scope_type TEXT DEFAULT 'agent',
+    scope_id TEXT,
+    source_type TEXT,
+    source_id TEXT,
+    source_label TEXT,
+    stale_after_days INTEGER,
+    metadata_json TEXT DEFAULT '{}',
     content TEXT NOT NULL,
     importance INTEGER DEFAULT 5,
     embedding TEXT,
@@ -503,6 +603,36 @@ db.exec(`
   CREATE INDEX IF NOT EXISTS idx_memories_user ON memories(user_id, archived, updated_at DESC);
   CREATE INDEX IF NOT EXISTS idx_memories_category ON memories(user_id, category, archived);
 
+  CREATE TABLE IF NOT EXISTS assistant_self_state (
+    user_id INTEGER NOT NULL,
+    agent_id TEXT NOT NULL,
+    identity_json TEXT DEFAULT '{}',
+    focus_json TEXT DEFAULT '{}',
+    updated_at TEXT DEFAULT (datetime('now')),
+    PRIMARY KEY (user_id, agent_id),
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
+    FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE CASCADE
+  );
+
+  CREATE TABLE IF NOT EXISTS agent_run_events (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    run_id TEXT NOT NULL,
+    user_id INTEGER NOT NULL,
+    agent_id TEXT,
+    event_type TEXT NOT NULL,
+    request_id TEXT,
+    step_id TEXT,
+    sequence_index INTEGER DEFAULT 0,
+    payload_json TEXT DEFAULT '{}',
+    created_at TEXT DEFAULT (datetime('now')),
+    FOREIGN KEY (run_id) REFERENCES agent_runs(id) ON DELETE CASCADE,
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
+    FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_agent_run_events_run ON agent_run_events(run_id, sequence_index, id);
+  CREATE INDEX IF NOT EXISTS idx_agent_run_events_user ON agent_run_events(user_id, created_at DESC);
+
   CREATE TABLE IF NOT EXISTS agent_settings (
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     user_id INTEGER NOT NULL,
@@ -651,6 +781,40 @@ db.exec(`
   CREATE INDEX IF NOT EXISTS idx_recording_chunks_source ON recording_chunks(source_id, sequence_index);
   CREATE INDEX IF NOT EXISTS idx_recording_segments_session ON recording_transcript_segments(session_id, start_ms, created_at);
 
+  CREATE TABLE IF NOT EXISTS screen_history (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id INTEGER NOT NULL,
+    timestamp TEXT DEFAULT (datetime('now')),
+    app_name TEXT,
+    text_content TEXT,
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+  );
+
+  CREATE TABLE IF NOT EXISTS notification_history (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id INTEGER NOT NULL,
+    app_package TEXT,
+    title TEXT,
+    body TEXT,
+    timestamp TEXT DEFAULT (datetime('now')),
+    action_taken TEXT,
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+  );
+
+  CREATE TABLE IF NOT EXISTS geofences (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    user_id INTEGER NOT NULL,
+    label TEXT,
+    latitude REAL NOT NULL,
+    longitude REAL NOT NULL,
+    radius_meters INTEGER NOT NULL,
+    trigger_action TEXT,
+    FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_screen_history_user ON screen_history(user_id, timestamp DESC);
+  CREATE INDEX IF NOT EXISTS idx_notification_history_user ON notification_history(user_id, timestamp DESC);
+
   CREATE TABLE IF NOT EXISTS artifacts (
     id TEXT PRIMARY KEY,
     user_id INTEGER NOT NULL,
@@ -676,6 +840,29 @@ db.exec(`
 
 try {
   db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS screen_history_fts USING fts5(
+      text_content,
+      app_name,
+      timestamp UNINDEXED,
+      user_id UNINDEXED,
+      tokenize = 'porter unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS screen_history_fts_ai AFTER INSERT ON screen_history BEGIN
+      INSERT INTO screen_history_fts(rowid, text_content, app_name, timestamp, user_id)
+      VALUES (new.id, COALESCE(new.text_content, ''), COALESCE(new.app_name, ''), new.timestamp, new.user_id);
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS screen_history_fts_ad AFTER DELETE ON screen_history BEGIN
+      DELETE FROM screen_history_fts WHERE rowid = old.id;
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS screen_history_fts_au AFTER UPDATE ON screen_history BEGIN
+      DELETE FROM screen_history_fts WHERE rowid = old.id;
+      INSERT INTO screen_history_fts(rowid, text_content, app_name, timestamp, user_id)
+      VALUES (new.id, COALESCE(new.text_content, ''), COALESCE(new.app_name, ''), new.timestamp, new.user_id);
+    END;
+
     CREATE VIRTUAL TABLE IF NOT EXISTS conversation_history_fts USING fts5(
       content,
       role UNINDEXED,
@@ -766,6 +953,16 @@ for (const col of [
   "ALTER TABLE conversation_history ADD COLUMN agent_id TEXT",
   "ALTER TABLE memories ADD COLUMN agent_id TEXT",
   "ALTER TABLE core_memory ADD COLUMN agent_id TEXT",
+  "ALTER TABLE ai_widgets ADD COLUMN widget_kind TEXT DEFAULT 'custom'",
+  "ALTER TABLE ai_widgets ADD COLUMN system_key TEXT",
+  "ALTER TABLE ai_widgets ADD COLUMN is_system INTEGER DEFAULT 0",
+  "ALTER TABLE memories ADD COLUMN scope_type TEXT DEFAULT 'agent'",
+  "ALTER TABLE memories ADD COLUMN scope_id TEXT",
+  "ALTER TABLE memories ADD COLUMN source_type TEXT",
+  "ALTER TABLE memories ADD COLUMN source_id TEXT",
+  "ALTER TABLE memories ADD COLUMN source_label TEXT",
+  "ALTER TABLE memories ADD COLUMN stale_after_days INTEGER",
+  "ALTER TABLE memories ADD COLUMN metadata_json TEXT DEFAULT '{}'",
   "ALTER TABLE scheduled_tasks ADD COLUMN run_at TEXT",
   "ALTER TABLE scheduled_tasks ADD COLUMN one_time INTEGER DEFAULT 0",
   "ALTER TABLE scheduled_tasks ADD COLUMN execution_mode TEXT DEFAULT 'prompt'",
@@ -818,12 +1015,15 @@ function createAgentScopedIndexes() {
   const statements = [
     ['agent_runs', 'CREATE INDEX IF NOT EXISTS idx_agent_runs_agent ON agent_runs(user_id, agent_id, created_at DESC)'],
     ['integration_connections', 'CREATE INDEX IF NOT EXISTS idx_integration_connections_agent ON integration_connections(user_id, agent_id, provider_key, app_key)'],
+    ['integration_provider_configs', 'CREATE INDEX IF NOT EXISTS idx_integration_provider_configs_agent ON integration_provider_configs(user_id, agent_id, provider_key)'],
     ['messages', 'CREATE INDEX IF NOT EXISTS idx_messages_agent ON messages(user_id, agent_id, created_at DESC)'],
     ['conversations', 'CREATE INDEX IF NOT EXISTS idx_conversations_agent ON conversations(user_id, agent_id, updated_at DESC)'],
     ['scheduled_tasks', 'CREATE INDEX IF NOT EXISTS idx_scheduled_tasks_agent ON scheduled_tasks(user_id, agent_id)'],
     ['conversation_history', 'CREATE INDEX IF NOT EXISTS idx_conv_history_agent ON conversation_history(user_id, agent_id, created_at DESC)'],
     ['memories', 'CREATE INDEX IF NOT EXISTS idx_memories_agent ON memories(user_id, agent_id, archived, updated_at DESC)'],
+    ['memories', 'CREATE INDEX IF NOT EXISTS idx_memories_scope ON memories(user_id, agent_id, scope_type, scope_id, archived, updated_at DESC)'],
     ['core_memory', 'CREATE INDEX IF NOT EXISTS idx_core_memory_agent ON core_memory(user_id, agent_id, key)'],
+    ['ai_widgets', 'CREATE UNIQUE INDEX IF NOT EXISTS idx_ai_widgets_system_key ON ai_widgets(user_id, agent_id, system_key) WHERE system_key IS NOT NULL'],
   ];
   for (const [table, statement] of statements) {
     if (!tableHasColumn(table, 'agent_id')) continue;
@@ -865,6 +1065,7 @@ function backfillAgentIds() {
     'platform_connections',
     'mcp_servers',
     'integration_connections',
+    'integration_provider_configs',
     'integration_oauth_states',
     'scheduled_tasks',
     'conversations',
@@ -884,7 +1085,19 @@ function backfillAgentIds() {
       }
     }
   });
-  tx();
+
+  try {
+    runWithBusyRetry(
+      () => tx(),
+      { attempts: 8, label: 'backfillAgentIds' },
+    );
+  } catch (error) {
+    if (isSqliteBusyError(error)) {
+      console.warn('[Database] Skipping backfillAgentIds because the database stayed locked.');
+      return;
+    }
+    throw error;
+  }
 }
 
 function backfillAgentPolicies() {
@@ -896,21 +1109,30 @@ function backfillAgentPolicies() {
     return;
   }
   try {
-    db.prepare(
-      `UPDATE agents
-       SET can_delegate = COALESCE(can_delegate, 1),
-           can_be_delegated_to = COALESCE(can_be_delegated_to, 0),
-           delegate_targets_json = COALESCE(delegate_targets_json, '[]')
-       WHERE slug = 'main'`
-    ).run();
-    db.prepare(
-      `UPDATE agents
-       SET can_delegate = COALESCE(can_delegate, 0),
-           can_be_delegated_to = COALESCE(can_be_delegated_to, 1),
-           delegate_targets_json = COALESCE(delegate_targets_json, '[]')
-       WHERE slug != 'main'`
-    ).run();
-  } catch {
+    runWithBusyRetry(
+      () => {
+        db.prepare(
+          `UPDATE agents
+           SET can_delegate = COALESCE(can_delegate, 1),
+               can_be_delegated_to = COALESCE(can_be_delegated_to, 0),
+               delegate_targets_json = COALESCE(delegate_targets_json, '[]')
+           WHERE slug = 'main'`
+        ).run();
+        db.prepare(
+          `UPDATE agents
+           SET can_delegate = COALESCE(can_delegate, 0),
+               can_be_delegated_to = COALESCE(can_be_delegated_to, 1),
+               delegate_targets_json = COALESCE(delegate_targets_json, '[]')
+           WHERE slug != 'main'`
+        ).run();
+      },
+      { attempts: 8, label: 'backfillAgentPolicies' },
+    );
+  } catch (error) {
+    if (isSqliteBusyError(error)) {
+      console.warn('[Database] Skipping backfillAgentPolicies because the database stayed locked.');
+      return;
+    }
     // Keep startup resilient for partially migrated databases.
   }
 }
@@ -1246,6 +1468,83 @@ function migrateIntegrationOauthStatesTable() {
   `);
 }
 
+function migrateIntegrationProviderConfigsTable() {
+  if (
+    tableHasColumn('integration_provider_configs', 'agent_id') &&
+    tableHasUniqueIndex('integration_provider_configs', ['user_id', 'agent_id', 'provider_key'])
+  ) {
+    const users = db.prepare('SELECT id FROM users').all();
+    for (const user of users) {
+      const agentId = getMainAgentId(user.id);
+      db.prepare(
+        'UPDATE integration_provider_configs SET agent_id = ? WHERE user_id = ? AND agent_id IS NULL'
+      ).run(agentId, user.id);
+    }
+    return;
+  }
+
+  const rows = db
+    .prepare('SELECT * FROM integration_provider_configs ORDER BY id ASC')
+    .all();
+
+  db.exec('BEGIN');
+  try {
+    db.exec(`
+      ALTER TABLE integration_provider_configs RENAME TO integration_provider_configs_legacy_agent_scope;
+
+      CREATE TABLE integration_provider_configs (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        user_id INTEGER NOT NULL,
+        agent_id TEXT,
+        provider_key TEXT NOT NULL,
+        config_json TEXT DEFAULT '{}',
+        created_at TEXT DEFAULT (datetime('now')),
+        updated_at TEXT DEFAULT (datetime('now')),
+        FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
+        FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE SET NULL,
+        UNIQUE(user_id, agent_id, provider_key)
+      );
+    `);
+    const insert = db.prepare(`
+      INSERT OR REPLACE INTO integration_provider_configs (
+        id,
+        user_id,
+        agent_id,
+        provider_key,
+        config_json,
+        created_at,
+        updated_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+
+    for (const row of rows) {
+      insert.run(
+        row.id,
+        row.user_id,
+        row.agent_id || getMainAgentId(row.user_id),
+        row.provider_key,
+        row.config_json,
+        row.created_at,
+        row.updated_at,
+      );
+    }
+
+    db.exec(`
+      DROP TABLE integration_provider_configs_legacy_agent_scope;
+      CREATE INDEX IF NOT EXISTS idx_integration_provider_configs_agent
+        ON integration_provider_configs(user_id, agent_id, provider_key);
+    `);
+    db.exec('COMMIT');
+  } catch (error) {
+    try {
+      db.exec('ROLLBACK');
+    } catch {
+      // Ignore rollback errors and rethrow the original migration failure.
+    }
+    throw error;
+  }
+}
+
 function migrateIntegrationSecretStorage() {
   try {
     const connectionRows = db
@@ -1278,6 +1577,22 @@ function migrateIntegrationSecretStorage() {
   } catch {
     // Preserve startup even if a row cannot be re-encrypted.
   }
+
+  try {
+    const providerRows = db
+      .prepare('SELECT id, config_json FROM integration_provider_configs')
+      .all();
+    const updateProviderConfig = db.prepare(
+      'UPDATE integration_provider_configs SET config_json = ? WHERE id = ?',
+    );
+    for (const row of providerRows) {
+      const current = String(row.config_json || '');
+      if (!current || isEncryptedValue(current)) continue;
+      updateProviderConfig.run(encryptValue(current), row.id);
+    }
+  } catch {
+    // Preserve startup even if a row cannot be re-encrypted.
+  }
 }
 
 function backfillVerifiedAccountEmails() {
@@ -1351,11 +1666,38 @@ rebuildPlatformConnectionsForAgents();
 rebuildCoreMemoryForAgents();
 migrateIntegrationConnectionsTable();
 migrateIntegrationOauthStatesTable();
+migrateIntegrationProviderConfigsTable();
 createAgentScopedIndexes();
 migrateIntegrationSecretStorage();
 backfillVerifiedAccountEmails();
 rebuildFtsForAgents();
 
+function migrateUsersOnboarding() {
+  try {
+    const columns = db.pragma('table_info(users)');
+    const hasOnboardingCol = columns.some((c) => c.name === 'has_completed_onboarding');
+    if (!hasOnboardingCol) {
+      db.exec('ALTER TABLE users ADD COLUMN has_completed_onboarding INTEGER DEFAULT 0');
+    }
+  } catch (err) {
+    console.warn('Could not add has_completed_onboarding column:', err.message);
+  }
+}
+migrateUsersOnboarding();
+
+function migrateUsersDisplayName() {
+  try {
+    const columns = db.pragma('table_info(users)');
+    const hasDisplayNameCol = columns.some((c) => c.name === 'display_name');
+    if (!hasDisplayNameCol) {
+      db.exec('ALTER TABLE users ADD COLUMN display_name TEXT');
+    }
+  } catch (err) {
+    console.warn('Could not add display_name column:', err.message);
+  }
+}
+migrateUsersDisplayName();
+
 try {
   db.exec(`
     INSERT OR REPLACE INTO conversation_history_fts(rowid, content, role, user_id, agent_id, agent_run_id)

package/server/guest-agent.android.package.json

@@ -0,0 +1,13 @@
+{
+  "name": "neoagent-guest-agent-android",
+  "private": true,
+  "version": "1.0.0",
+  "description": "Guest runtime for NeoAgent Android and CLI services",
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "express": "^4.21.2",
+    "proper-lockfile": "^4.1.2"
+  }
+}

package/server/guest-agent.browser.package.json

@@ -0,0 +1,14 @@
+{
+  "name": "neoagent-guest-agent-browser",
+  "private": true,
+  "version": "1.0.0",
+  "description": "Guest runtime for NeoAgent browser and CLI services",
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "express": "^4.21.2",
+    "playwright-chromium": "^1.59.1",
+    "proper-lockfile": "^4.1.2"
+  }
+}