neoagent 2.3.0 → 2.3.1-beta.10

package/server/services/messaging/manager.js

@@ -33,6 +33,7 @@ const {
   summarizeAccessPolicy,
   classifyRecentTarget,
 } = require('./access_policy');
+const { decryptValue, encryptValue } = require('../integrations/secrets');
 
 const LEGACY_WHATSAPP_AUTH_DIR = path.join(DATA_DIR, 'whatsapp-auth');
 
@@ -268,6 +269,31 @@ class MessagingManager extends EventEmitter {
     return undefined;
   }
 
+  _encodeStoredConfig(config) {
+    const serialized = JSON.stringify(this._persistableConfig(config) || {});
+    if (!serialized) return '{}';
+    try {
+      return encryptValue(serialized);
+    } catch {
+      return serialized;
+    }
+  }
+
+  _decodeStoredConfig(value) {
+    const raw = String(value || '').trim();
+    if (!raw) return {};
+    try {
+      const decoded = decryptValue(raw);
+      return decoded ? JSON.parse(decoded) : {};
+    } catch {
+      try {
+        return JSON.parse(raw);
+      } catch {
+        return {};
+      }
+    }
+  }
+
   async connectPlatform(userId, platformName, config = {}, options = {}) {
     const agentId = this._agentId(userId, options);
     config = { ...(config || {}) };
@@ -322,7 +348,7 @@ class MessagingManager extends EventEmitter {
       config.voiceRuntimeManager = this.voiceRuntimeManager || null;
     }
 
-    const storedConfig = JSON.stringify(this._persistableConfig(config) || {});
+    const storedConfig = this._encodeStoredConfig(config);
 
     const key = this._key(userId, agentId, platformName);
     let platform = this.platforms.get(key);
@@ -569,11 +595,7 @@ class MessagingManager extends EventEmitter {
     if (platformName === 'whatsapp') {
       reconnectConfig = platform?.config || {};
       if ((!reconnectConfig || Object.keys(reconnectConfig).length === 0) && row?.config) {
-        try {
-          reconnectConfig = JSON.parse(row.config);
-        } catch {
-          reconnectConfig = {};
-        }
+        reconnectConfig = this._decodeStoredConfig(row.config);
       }
     }
     if (platform && platform.logout) {
@@ -594,7 +616,7 @@ class MessagingManager extends EventEmitter {
     ).all();
     for (const row of rows) {
       try {
-        const config = row.config ? JSON.parse(row.config) : {};
+        const config = this._decodeStoredConfig(row.config);
         console.log(`[Messaging] Restoring ${row.platform} for user ${row.user_id} agent ${row.agent_id || 'main'}`);
         await this.connectPlatform(row.user_id, row.platform, config, { agentId: row.agent_id });
       } catch (err) {

package/server/services/skills/base_catalog.js

@@ -1161,6 +1161,39 @@ Auto-orients and arranges the STL, applies settings from JSON files, slices all
 3. Build the command from the flags above
 4. Run it and check for errors in the output (debug level 2 is a good default)
 5. Report the output file location and any warnings`
+  },
+
+  {
+    id: 'package-tracker',
+    name: 'Package Tracker',
+    description: 'Track packages by automatically detecting the carrier and fetching the tracking link.',
+    category: 'info',
+    icon: '📦',
+    content: `---
+name: package-tracker
+description: Track packages by automatically detecting the carrier and fetching the tracking link.
+category: info
+icon: 📦
+enabled: true
+---
+
+You are a package tracking assistant. When the user gives you a tracking number, you should:
+
+1. Identify the carrier using common regex patterns:
+   - UPS: \`\\b(1Z[0-9A-Z]{16})\\b\`
+   - FedEx: \`\\b([0-9]{12,15})\\b\` (usually 12 or 15 digits)
+   - USPS: \`\\b(94[0-9]{20})\\b\` or \`\\b([A-Z]{2}[0-9]{9}US)\\b\`
+   - DHL: \`\\b([0-9]{10})\\b\` or \`\\b([0-9]{20})\\b\`
+
+2. Generate the direct tracking link for the user:
+   - UPS: \`https://www.ups.com/track?tracknum={number}\`
+   - FedEx: \`https://www.fedex.com/fedextrack/?trknbr={number}\`
+   - USPS: \`https://tools.usps.com/go/TrackConfirmAction?tLabels={number}\`
+   - DHL: \`https://www.dhl.com/global-en/home/tracking/tracking-express.html?submit=1&tracking-id={number}\`
+
+3. If the carrier is not obvious, use \`web_search\` with the query "track package {number}" to identify the carrier.
+4. Present the carrier and the direct tracking link to the user clearly.
+5. Optionally use \`browser_navigate\` or \`http_request\` to fetch the current status from the tracking link, but note that many carriers block automated scraping. The direct link is the most important output.`
   }
 ];
 

package/server/services/tasks/adapters/index.js

@@ -2,9 +2,11 @@
 
 module.exports = [
   require('./schedule'),
+  require('./manual'),
   require('./gmail_message_received'),
   require('./outlook_email_received'),
   require('./slack_message_received'),
   require('./teams_message_received'),
+  require('./weather_event'),
   require('./whatsapp_personal_message_received'),
 ];

package/server/services/tasks/adapters/manual.js

@@ -0,0 +1,12 @@
+'use strict';
+
+module.exports = {
+  type: 'manual',
+  label: 'Manual Trigger',
+  async validateConfig() {
+    return {};
+  },
+  summarize() {
+    return 'Manual run only';
+  },
+};

package/server/services/tasks/adapters/schedule.js

@@ -1,7 +1,7 @@
 'use strict';
 
 const cron = require('node-cron');
-const { findNextRun } = require('../schedule_utils');
+const { findNextRun, parseCronExpression } = require('../schedule_utils');
 
 function normalizeRunAt(value) {
   if (!value) return null;
@@ -12,6 +12,37 @@ function normalizeRunAt(value) {
   return date.toISOString();
 }
 
+function normalizeCronExpression(value) {
+  const raw = String(value || '').trim();
+  if (!raw) {
+    throw new Error('A valid cron expression is required.');
+  }
+
+  const fields = raw.split(/\s+/);
+  if (fields.length === 6) {
+    const seconds = String(fields[0] || '').trim();
+    if (seconds !== '0' && seconds !== '*') {
+      throw new Error('Cron expressions with seconds are not supported. Use a 5-field expression.');
+    }
+    fields.shift();
+  }
+
+  if (fields.length !== 5) {
+    throw new Error('A valid cron expression is required.');
+  }
+
+  // Quartz-style "?" means "no specific value"; convert to standard wildcard.
+  if (fields[2] === '?') fields[2] = '*';
+  if (fields[4] === '?') fields[4] = '*';
+
+  const normalized = fields.join(' ');
+  parseCronExpression(normalized);
+  if (!cron.validate(normalized)) {
+    throw new Error('A valid cron expression is required.');
+  }
+  return normalized;
+}
+
 module.exports = {
   type: 'schedule',
   label: 'Schedule',
@@ -31,10 +62,7 @@ module.exports = {
       };
     }
 
-    const cronExpression = String(config.cronExpression || config.cron_expression || '').trim();
-    if (!cronExpression || !cron.validate(cronExpression)) {
-      throw new Error('A valid cron expression is required.');
-    }
+    const cronExpression = normalizeCronExpression(config.cronExpression || config.cron_expression);
     return {
       mode,
       cronExpression,

package/server/services/tasks/adapters/weather_event.js

@@ -0,0 +1,84 @@
+'use strict';
+
+const {
+  ensureOwnedIntegrationConnection,
+  normalizeTrimmedText,
+} = require('../security');
+
+const WEATHER_EVENT_TYPES = new Set([
+  'rain_start',
+  'snow_start',
+  'wind_alert',
+  'temperature_above',
+  'temperature_below',
+]);
+
+function normalizeNumber(value, fallback) {
+  const numeric = Number(value);
+  return Number.isFinite(numeric) ? numeric : fallback;
+}
+
+function normalizeEventTypes(value) {
+  const raw = Array.isArray(value)
+    ? value
+    : String(value || '')
+        .split(',')
+        .map((entry) => entry.trim())
+        .filter(Boolean);
+  const normalized = raw
+    .map((entry) => String(entry || '').trim().toLowerCase())
+    .filter((entry) => WEATHER_EVENT_TYPES.has(entry));
+  return Array.from(new Set(normalized));
+}
+
+module.exports = {
+  type: 'weather_event',
+  label: 'Weather Event',
+  providerKey: 'weather',
+  appKey: 'forecast',
+  async validateConfig(config = {}, context = {}) {
+    const connection = ensureOwnedIntegrationConnection(context.integrationManager, {
+      userId: context.userId,
+      agentId: context.agentId,
+      connectionId: config.connectionId || config.connection_id,
+      providerKey: 'weather',
+      appKey: 'forecast',
+    });
+
+    const eventTypes = normalizeEventTypes(config.eventTypes || config.event_types);
+    if (eventTypes.length === 0) {
+      throw new Error(
+        'At least one weather event type is required: rain_start, snow_start, wind_alert, temperature_above, temperature_below.',
+      );
+    }
+
+    const location = normalizeTrimmedText(
+      config.location || config.locationQuery || config.query,
+      180,
+    );
+    if (!location) {
+      throw new Error('Weather event location is required (for example: Berlin, DE).');
+    }
+
+    return {
+      connectionId: connection.id,
+      accountEmail: connection.account_email || null,
+      location,
+      eventTypes,
+      minPrecipitationMm: normalizeNumber(config.minPrecipitationMm ?? config.min_precipitation_mm, 0.4),
+      minSnowfallCm: normalizeNumber(config.minSnowfallCm ?? config.min_snowfall_cm, 0.2),
+      windAlertKph: normalizeNumber(config.windAlertKph ?? config.wind_alert_kph, 40),
+      temperatureAboveC: normalizeNumber(config.temperatureAboveC ?? config.temperature_above_c, 32),
+      temperatureBelowC: normalizeNumber(config.temperatureBelowC ?? config.temperature_below_c, 0),
+      horizonHours: Math.max(1, Math.min(Number(config.horizonHours || config.horizon_hours) || 12, 48)),
+    };
+  },
+  summarize(config = {}) {
+    const parts = ['Weather'];
+    if (config.location) parts.push(config.location);
+    if (Array.isArray(config.eventTypes) && config.eventTypes.length > 0) {
+      parts.push(config.eventTypes.join(', '));
+    }
+    return parts.join(' · ');
+  },
+};

package/server/services/tasks/integration_runtime.js

@@ -8,6 +8,7 @@ const POLLED_TRIGGER_TYPES = Object.freeze([
   'outlook_email_received',
   'slack_message_received',
   'teams_message_received',
+  'weather_event',
   'whatsapp_personal_message_received',
 ]);
 
@@ -166,6 +167,90 @@ async function fetchTriggerRows({ integrationManager, userId, agentId, triggerTy
       .sort(sortByTimestamp);
   }
 
+  if (triggerType === 'weather_event') {
+    const forecast = await integrationManager.executeTool(userId, 'weather_get_forecast', {
+      ...connectionArg,
+      ...(config.location ? { location: config.location } : {}),
+      forecast_hours: Math.max(1, Math.min(Number(config.horizonHours) || 12, 48)),
+    }, scopedAgentId);
+    const hourly = Array.isArray(forecast?.hourly) ? forecast.hourly : [];
+    const eventTypes = Array.isArray(config.eventTypes) ? config.eventTypes : [];
+    const rows = [];
+
+    for (let index = 0; index < hourly.length; index += 1) {
+      const row = hourly[index] || {};
+      const previous = index > 0 ? (hourly[index - 1] || {}) : null;
+      const time = String(row.time || '').trim();
+      if (!time) continue;
+
+      const rain = Number(row.rain || row.precipitation || 0);
+      const prevRain = Number(previous?.rain || previous?.precipitation || 0);
+      const snowfall = Number(row.snowfall || 0);
+      const prevSnow = Number(previous?.snowfall || 0);
+      const windSpeed = Number(row.windSpeed || 0);
+      const temperature = Number(row.temperature);
+
+      const candidates = [
+        {
+          type: 'rain_start',
+          active:
+            eventTypes.includes('rain_start')
+            && rain >= Number(config.minPrecipitationMm || 0.4)
+            && prevRain < Number(config.minPrecipitationMm || 0.4),
+        },
+        {
+          type: 'snow_start',
+          active:
+            eventTypes.includes('snow_start')
+            && snowfall >= Number(config.minSnowfallCm || 0.2)
+            && prevSnow < Number(config.minSnowfallCm || 0.2),
+        },
+        {
+          type: 'wind_alert',
+          active:
+            eventTypes.includes('wind_alert')
+            && windSpeed >= Number(config.windAlertKph || 40),
+        },
+        {
+          type: 'temperature_above',
+          active:
+            eventTypes.includes('temperature_above')
+            && Number.isFinite(temperature)
+            && temperature >= Number(config.temperatureAboveC || 32),
+        },
+        {
+          type: 'temperature_below',
+          active:
+            eventTypes.includes('temperature_below')
+            && Number.isFinite(temperature)
+            && temperature <= Number(config.temperatureBelowC || 0),
+        },
+      ];
+
+      for (const candidate of candidates) {
+        if (!candidate.active) continue;
+        rows.push({
+          fingerprint: `weather:${config.connectionId}:${candidate.type}:${time}`,
+          timestamp: time,
+          context: {
+            triggerEvent: {
+              provider: 'weather',
+              eventType: candidate.type,
+              location: forecast?.location?.label || config.location || null,
+              time,
+              rain,
+              snowfall,
+              windSpeed,
+              temperature,
+            },
+          },
+        });
+      }
+    }
+
+    return rows.sort(sortByTimestamp);
+  }
+
   return [];
 }
 

package/server/services/tasks/runtime.js

@@ -66,7 +66,7 @@ class TaskRuntime {
       label: adapter.label,
       providerKey: adapter.providerKey || null,
       appKey: adapter.appKey || null,
-      available: adapter.type === 'schedule'
+      available: adapter.type === 'schedule' || adapter.type === 'manual'
         ? true
         : this._hasConnectedApp(userId, agentId, adapter.providerKey, adapter.appKey),
     }));
@@ -222,8 +222,8 @@ class TaskRuntime {
     const existing = this.scheduleJobs.get(taskId);
     if (existing) {
       existing.task.stop();
-      this.scheduleJobs.delete(taskId);
     }
+    this.scheduleJobs.delete(taskId);
   }
 
   async _executeTask(taskId, userId, executionMeta = {}) {

package/server/services/voice/agentBridge.js

@@ -23,6 +23,7 @@ class VoiceAgentBridge {
     session.currentRunId = runId;
 
     try {
+      const deferredFollowUp = await this.runtimeManager.prepareDeferredVoiceFollowUp(session);
       const result = await runVoiceTranscriptTurn({
         userId: session.userId,
         agentId: session.agentId,
@@ -41,10 +42,25 @@ class VoiceAgentBridge {
         runId,
       });
       session.currentRunId = result.runId || runId;
-      if (String(result.replyText || '').trim()) {
-        await this.runtimeManager.deliverAssistantMessage(session, result.replyText, {
-          kind: 'final',
-        });
+      const replyText = String(result.replyText || '').trim();
+      if (replyText) {
+        if (deferredFollowUp) {
+          const followUp = await this.runtimeManager.deliverDeferredVoiceFollowUp(
+            session,
+            deferredFollowUp,
+            replyText,
+            result.runId || runId,
+          );
+          if (!followUp?.sent) {
+            await this.runtimeManager.deliverAssistantMessage(session, replyText, {
+              kind: 'final',
+            });
+          }
+        } else {
+          await this.runtimeManager.deliverAssistantMessage(session, replyText, {
+            kind: 'final',
+          });
+        }
       }
       await session.setState('idle');
       session.currentRunId = null;

package/server/services/voice/message.js

@@ -61,6 +61,9 @@ function buildDirectVoiceContext({
     allowInterimUpdates
       ? 'Do not use send_message. Use send_interim_update only for short spoken progress updates when silence would otherwise be noticeable.'
       : 'Do not use send_message or send_interim_update.',
+    allowInterimUpdates
+      ? 'You decide when a task is taking long. If so, send a natural deferral via send_interim_update and set defer_follow_up=true.'
+      : 'Keep the spoken response direct and concise.',
     'Return only the assistant reply.',
   ];
 

package/server/services/voice/openaiClient.js

@@ -4,6 +4,7 @@ const fs = require('fs');
 const path = require('path');
 const { OpenAI } = require('openai');
 const { AGENT_DATA_DIR } = require('../../../runtime/paths');
+const { decryptLocalValue } = require('../../utils/local_secrets');
 
 let cachedClient = null;
 
@@ -15,7 +16,9 @@ function resolveOpenAiApiKey() {
   try {
     const keysPath = path.join(AGENT_DATA_DIR, 'API_KEYS.json');
     const keys = JSON.parse(fs.readFileSync(keysPath, 'utf8'));
-    const candidate = keys.OPENAI_API_KEY || keys.openai_api_key || keys.openai;
+    const candidate = decryptLocalValue(keys.OPENAI_API_KEY)
+      || decryptLocalValue(keys.openai_api_key)
+      || decryptLocalValue(keys.openai);
     return typeof candidate === 'string' && candidate.trim() ? candidate.trim() : '';
   } catch {
     return '';

package/server/services/voice/providers.js

@@ -6,6 +6,7 @@ const { AGENT_DATA_DIR } = require('../../../runtime/paths');
 const { getOpenAiClient } = require('./openaiClient');
 const { synthesizeSpeechBuffer } = require('./openaiSpeech');
 const { transcribeChunkWithDeepgram } = require('../recordings/deepgram');
+const { decryptLocalValue } = require('../../utils/local_secrets');
 
 const DEFAULT_STT_PROVIDER = 'openai';
 const DEFAULT_TTS_PROVIDER = 'openai';
@@ -94,7 +95,7 @@ function resolveApiKey(candidates = []) {
     const snake = lower.replace(/[^a-z0-9]+/g, '_');
     const variants = [key, lower, snake];
     for (const variant of variants) {
-      const value = keys[variant];
+      const value = decryptLocalValue(keys[variant]);
       if (typeof value === 'string' && value.trim()) {
         return value.trim();
       }

package/server/services/voice/runtimeManager.js

@@ -1,8 +1,10 @@
 'use strict';
 
 const { randomUUID } = require('crypto');
+const db = require('../../db/database');
 
 const { getProviderRuntimeConfig } = require('../ai/models');
+const { isMainAgent, resolveAgentId } = require('../agents/manager');
 const { getVoiceRuntimeSettings } = require('./liveSettings');
 const { VoiceLiveSession } = require('./liveSession');
 const { OpenAiLiveRelayAdapter } = require('./openaiLiveRelayAdapter');
@@ -180,11 +182,14 @@ class VoiceRuntimeManager {
     await session.setState('idle');
   }
 
-  async publishInterimUpdate({ sessionId, content, kind = 'progress' } = {}) {
+  async publishInterimUpdate({ sessionId, content, kind = 'progress', deferFollowUp = false } = {}) {
     const session = this.getSession(sessionId);
     if (!session || session.closed) {
       return { sent: false, skipped: true, reason: 'Voice session is not active.' };
     }
+    if (deferFollowUp === true) {
+      session.deferFollowUpRequested = true;
+    }
     await this.deliverAssistantMessage(session, content, { kind });
     return { sent: true };
   }
@@ -196,6 +201,88 @@ class VoiceRuntimeManager {
     await session.publishAssistantOutput(normalized, options);
   }
 
+  async prepareDeferredVoiceFollowUp(session) {
+    if (!session || session.closed) return null;
+    if (String(session.platform || '').trim() !== 'voice_live') return null;
+
+    const target = this.#resolvePreferredMessagingTarget(session.userId, session.agentId);
+    if (!target) return null;
+    return {
+      target,
+    };
+  }
+
+  async deliverDeferredVoiceFollowUp(session, followUpPlan, replyText, runId = null) {
+    if (!session || session.closed || !followUpPlan || session.deferFollowUpRequested !== true) {
+      return { sent: false, skipped: true };
+    }
+    const manager = this.#messagingManager();
+    if (!manager || typeof manager.sendMessage !== 'function') {
+      return { sent: false, skipped: true, reason: 'Messaging manager unavailable.' };
+    }
+
+    const target = followUpPlan.target;
+    if (!target?.platform || !target?.to) {
+      return { sent: false, skipped: true, reason: 'No deferred follow-up target.' };
+    }
+
+    const status = typeof manager.getPlatformStatus === 'function'
+      ? manager.getPlatformStatus(session.userId, target.platform, { agentId: session.agentId })
+      : null;
+    if (status && status.status !== 'connected') {
+      return { sent: false, skipped: true, reason: `Platform ${target.platform} is not connected.` };
+    }
+
+    const body = String(replyText || '').trim();
+    if (!body) {
+      return { sent: false, skipped: true, reason: 'Reply text is empty.' };
+    }
+
+    const followUpContent = [
+      'Update from your voice request:',
+      '',
+      body,
+    ].join('\n');
+
+    let sendResult;
+    try {
+      sendResult = await manager.sendMessage(
+        session.userId,
+        target.platform,
+        target.to,
+        followUpContent,
+        {
+          agentId: session.agentId,
+          runId,
+          persistConversation: true,
+        },
+      );
+    } catch (err) {
+      console.error('Failed to send deferred voice follow-up:', err);
+      return {
+        sent: false,
+        skipped: false,
+        success: false,
+        error: err instanceof Error ? err.message : String(err),
+        runId,
+      };
+    }
+
+    session.deferFollowUpRequested = false;
+    const label = this.#platformLabel(target.platform);
+    await this.deliverAssistantMessage(
+      session,
+      `I sent the full result to your ${label} chat.`,
+      { kind: 'final' },
+    );
+
+    return {
+      sent: true,
+      target,
+      result: sendResult,
+    };
+  }
+
   async startTelnyxTurn({
     userId,
     agentId = null,
@@ -284,6 +371,54 @@ class VoiceRuntimeManager {
     }
   }
 
+  #messagingManager() {
+    return this.agentEngine?.messagingManager || this.agentEngine?.app?.locals?.messagingManager || null;
+  }
+
+  #readScopedSetting(userId, agentId, key) {
+    const row = db.prepare(
+      'SELECT value FROM agent_settings WHERE user_id = ? AND agent_id = ? AND key = ?'
+    ).get(userId, agentId, key);
+    if (row) {
+      try {
+        return JSON.parse(row.value);
+      } catch {
+        return row.value;
+      }
+    }
+    if (!isMainAgent(userId, agentId)) return null;
+    const userRow = db.prepare(
+      'SELECT value FROM user_settings WHERE user_id = ? AND key = ?'
+    ).get(userId, key);
+    if (!userRow) return null;
+    try {
+      return JSON.parse(userRow.value);
+    } catch {
+      return userRow.value;
+    }
+  }
+
+  #resolvePreferredMessagingTarget(userId, agentId = null) {
+    const manager = this.#messagingManager();
+    if (!manager) return null;
+    const scopedAgentId = resolveAgentId(userId, agentId);
+    const platform = String(this.#readScopedSetting(userId, scopedAgentId, 'last_platform') || '').trim();
+    const to = String(this.#readScopedSetting(userId, scopedAgentId, 'last_chat_id') || '').trim();
+    if (!platform || !to) return null;
+
+    const status = typeof manager.getPlatformStatus === 'function'
+      ? manager.getPlatformStatus(userId, platform, { agentId: scopedAgentId })
+      : null;
+    if (!status || status.status !== 'connected') return null;
+    return { platform, to };
+  }
+
+  #platformLabel(platformName) {
+    const raw = String(platformName || '').trim();
+    if (!raw) return 'message';
+    return raw.replace(/[_-]+/g, ' ');
+  }
+
   async #deliverFlutterAssistantOutput(socket, sessionId, session, content, options = {}) {
     const kind = String(options.kind || 'final').trim() || 'final';
     socket.emit('voice:assistant_text', {