neoagent 3.1.1-beta.9 → 3.2.0

package/flutter_app/third_party/desktop_audio_capture/windows/system_audio_capture_plugin.h

@@ -1,87 +0,0 @@
-#ifndef FLUTTER_PLUGIN_SYSTEM_AUDIO_CAPTURE_PLUGIN_H_
-#define FLUTTER_PLUGIN_SYSTEM_AUDIO_CAPTURE_PLUGIN_H_
-
-#include <flutter/event_channel.h>
-#include <flutter/method_channel.h>
-#include <flutter/plugin_registrar_windows.h>
-#include <flutter/standard_method_codec.h>
-
-#include <memory>
-#include <mutex>
-#include <thread>
-#include <atomic>
-#include <vector>
-
-// Include Windows headers for WAVEFORMATEX
-#include <mmsystem.h>
-
-// Forward declarations for WASAPI interfaces
-struct IAudioClient;
-struct IAudioCaptureClient;
-struct IMMDevice;
-
-namespace audio_capture {
-
-class SystemAudioCapturePlugin : public flutter::Plugin {
- public:
-  static void RegisterWithRegistrar(flutter::PluginRegistrarWindows *registrar);
-
-  SystemAudioCapturePlugin(flutter::PluginRegistrarWindows *registrar);
-  ~SystemAudioCapturePlugin();
-
-  // Disallow copy and assign.
-  SystemAudioCapturePlugin(const SystemAudioCapturePlugin&) = delete;
-  SystemAudioCapturePlugin& operator=(const SystemAudioCapturePlugin&) = delete;
-
-  void HandleMethodCall(
-      const flutter::MethodCall<flutter::EncodableValue> &method_call,
-      std::unique_ptr<flutter::MethodResult<flutter::EncodableValue>> result);
-
- private:
-  bool StartCapture(const flutter::EncodableMap* args);
-  bool StopCapture();
-  void CaptureThread();
-  void SetThreadPriority();
-  double CalculateDecibel(const int16_t* samples, size_t sample_count);
-  void ApplyGainBoostAndConvertToMono(const int16_t* input, int16_t* output,
-                                      size_t frame_count, int input_channels,
-                                      float gain_boost);
-  void SendStatusUpdate(bool is_active);
-  void SendDecibelUpdate(double decibel);
-
-  flutter::PluginRegistrarWindows* registrar_;
-  std::unique_ptr<flutter::MethodChannel<flutter::EncodableValue>> method_channel_;
-  std::unique_ptr<flutter::EventChannel<flutter::EncodableValue>> event_channel_;
-  std::unique_ptr<flutter::EventChannel<flutter::EncodableValue>> status_event_channel_;
-  std::unique_ptr<flutter::EventChannel<flutter::EncodableValue>> decibel_event_channel_;
-  
-  std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> event_sink_;
-  std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> status_event_sink_;
-  std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> decibel_event_sink_;
-
-  std::mutex mutex_;
-  std::atomic<bool> is_capturing_;
-  std::atomic<bool> should_stop_;
-  std::thread capture_thread_;
-  
-  // Audio configuration
-  int sample_rate_;
-  int channels_;
-  int bits_per_sample_;
-  int chunk_duration_ms_;
-  float gain_boost_;
-  float input_volume_;
-  
-  // WASAPI interfaces
-  IAudioClient* audio_client_;
-  IAudioCaptureClient* capture_client_;
-  IMMDevice* device_;
-  WAVEFORMATEX* mix_format_;
-  UINT32 buffer_frame_count_;
-  bool com_initialized_;  // Track if we initialized COM
-};
-
-}  // namespace audio_capture
-
-#endif  // FLUTTER_PLUGIN_SYSTEM_AUDIO_CAPTURE_PLUGIN_H_
-

package/server/routes/recordings.js

@@ -1,335 +0,0 @@
-const express = require('express');
-const fs = require('fs');
-const { finished } = require('stream/promises');
-
-const { requireAuth } = require('../middleware/auth');
-const { sanitizeError } = require('../utils/security');
-const db = require('../db/database');
-const { readChunkBody } = require('./_helpers/readChunkBody');
-
-const router = express.Router();
-
-router.use(requireAuth);
-
-function getRecordingManager(req) {
-  return req.app.locals.recordingManager;
-}
-
-function parsePositiveInt(value, fallback) {
-  const parsed = Number.parseInt(value, 10);
-  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
-}
-
-function getChunkMetadata(req) {
-  const parseNonNegativeNumber = (value, fieldName) => {
-    const parsed = Number(value);
-    if (!Number.isFinite(parsed) || parsed < 0) {
-      throw new Error(`${fieldName} must be a non-negative number`);
-    }
-    return parsed;
-  };
-
-  const sequenceIndexRaw = req.get('x-recording-sequence') || req.query.sequenceIndex;
-  const startMsRaw = req.get('x-recording-start-ms') || req.query.startMs;
-  const endMsRaw = req.get('x-recording-end-ms') || req.query.endMs;
-  const mimeRaw = req.get('content-type') || req.query.mimeType || '';
-
-  return {
-    sourceKey: req.get('x-recording-source-key') || req.query.sourceKey,
-    sequenceIndex: parseNonNegativeNumber(sequenceIndexRaw, 'sequenceIndex'),
-    startMs: parseNonNegativeNumber(startMsRaw, 'startMs'),
-    endMs: parseNonNegativeNumber(endMsRaw, 'endMs'),
-    mimeType: String(mimeRaw).split(';')[0].trim(),
-  };
-}
-
-const WAV_HEADER_BYTES = 44;
-
-// Reads the canonical 44-byte WAV header our recorders emit. Returns the audio
-// format fields, or null when the file is not a canonical WAV (in which case
-// the caller streams the chunks unmodified instead of rebuilding a WAV).
-async function readCanonicalWavHeader(filePath) {
-  let handle;
-  try {
-    handle = await fs.promises.open(filePath, 'r');
-    const { bytesRead, buffer } = await handle.read(
-      Buffer.alloc(WAV_HEADER_BYTES),
-      0,
-      WAV_HEADER_BYTES,
-      0,
-    );
-    if (
-      bytesRead < WAV_HEADER_BYTES
-      || buffer.toString('ascii', 0, 4) !== 'RIFF'
-      || buffer.toString('ascii', 8, 12) !== 'WAVE'
-      || buffer.toString('ascii', 36, 40) !== 'data'
-    ) {
-      return null;
-    }
-    return {
-      audioFormat: buffer.readUInt16LE(20),
-      channelCount: buffer.readUInt16LE(22),
-      sampleRate: buffer.readUInt32LE(24),
-      bitsPerSample: buffer.readUInt16LE(34),
-    };
-  } catch {
-    return null;
-  } finally {
-    await handle?.close();
-  }
-}
-
-function buildWavHeader({ audioFormat, channelCount, sampleRate, bitsPerSample }, dataLength) {
-  const blockAlign = channelCount * (bitsPerSample / 8);
-  const byteRate = sampleRate * blockAlign;
-  const header = Buffer.alloc(WAV_HEADER_BYTES);
-  header.write('RIFF', 0, 4, 'ascii');
-  header.writeUInt32LE(36 + dataLength, 4);
-  header.write('WAVE', 8, 4, 'ascii');
-  header.write('fmt ', 12, 4, 'ascii');
-  header.writeUInt32LE(16, 16);
-  header.writeUInt16LE(audioFormat, 20);
-  header.writeUInt16LE(channelCount, 22);
-  header.writeUInt32LE(sampleRate, 24);
-  header.writeUInt32LE(byteRate, 28);
-  header.writeUInt16LE(blockAlign, 32);
-  header.writeUInt16LE(bitsPerSample, 34);
-  header.write('data', 36, 4, 'ascii');
-  header.writeUInt32LE(dataLength, 40);
-  return header;
-}
-
-function buildInlineFilename(sourceKey) {
-  const original = `${String(sourceKey || 'recording').normalize('NFKC')}.audio`
-    .replace(/[\r\n]+/g, ' ')
-    .trim() || 'recording.audio';
-  const safeAscii = original
-    .replace(/["]/g, "'")
-    .replace(/[^\x20-\x7E]+/g, '_')
-    .trim() || 'recording.audio';
-  return `inline; filename="${safeAscii}"; filename*=UTF-8''${encodeURIComponent(original)}`;
-}
-
-function statusFromMessage(message, rules, fallbackStatus = 500) {
-  for (const rule of rules) {
-    if (rule.pattern.test(message)) {
-      return rule.status;
-    }
-  }
-  return fallbackStatus;
-}
-
-function respondWithMappedError(res, err, rules, fallbackStatus = 500) {
-  const message = sanitizeError(err);
-  res.status(statusFromMessage(message, rules, fallbackStatus)).json({ error: message });
-}
-
-router.get('/', (req, res) => {
-  try {
-    const manager = getRecordingManager(req);
-    const sessions = manager.listSessions(req.session.userId, {
-      limit: parsePositiveInt(req.query.limit, 24),
-    });
-    res.json(sessions);
-  } catch (err) {
-    res.status(500).json({ error: sanitizeError(err) });
-  }
-});
-
-router.get('/:sessionId', (req, res) => {
-  try {
-    const manager = getRecordingManager(req);
-    const session = manager.getSession(req.session.userId, req.params.sessionId);
-    res.json({ session });
-  } catch (err) {
-    respondWithMappedError(res, err, [
-      { pattern: /not found/i, status: 404 },
-    ]);
-  }
-});
-
-router.get('/:sessionId/audio/:sourceKey', async (req, res) => {
-  try {
-    const sessionId = req.params.sessionId;
-    const sourceKey = String(req.params.sourceKey || '').trim();
-    if (!sourceKey) {
-      return res.status(400).json({ error: 'sourceKey is required.' });
-    }
-
-    const session = db.prepare(`
-      SELECT id
-      FROM recording_sessions
-      WHERE id = ? AND user_id = ?
-    `).get(sessionId, req.session.userId);
-    if (!session) {
-      return res.status(404).json({ error: 'Recording session not found.' });
-    }
-
-    const source = db.prepare(`
-      SELECT id, source_key, mime_type
-      FROM recording_sources
-      WHERE session_id = ? AND LOWER(source_key) = LOWER(?)
-      LIMIT 1
-    `).get(sessionId, sourceKey);
-    if (!source) {
-      return res.status(404).json({ error: 'Recording source not found.' });
-    }
-
-    const chunks = db.prepare(`
-      SELECT file_path, mime_type
-      FROM recording_chunks
-      WHERE source_id = ?
-      ORDER BY sequence_index ASC
-    `).all(source.id);
-    if (!Array.isArray(chunks) || chunks.length == 0) {
-      return res.status(404).json({ error: 'No audio chunks available.' });
-    }
-
-    const mimeType = String(source.mime_type || chunks[0]?.mime_type || 'application/octet-stream');
-    if (!mimeType.startsWith('audio/')) {
-      return res.status(415).json({
-        error: `Playback unsupported for mime type: ${mimeType}`,
-      });
-    }
-
-    const readableChunks = [];
-    for (const chunk of chunks) {
-      const filePath = chunk.file_path;
-      if (!filePath) {
-        continue;
-      }
-      try {
-        await fs.promises.access(filePath, fs.constants.R_OK);
-        readableChunks.push(chunk);
-      } catch {
-        // Skip chunk files that are missing or unreadable on disk.
-      }
-    }
-    if (readableChunks.length === 0) {
-      return res.status(404).json({ error: 'No audio chunks available.' });
-    }
-
-    res.setHeader('Content-Type', mimeType);
-    res.setHeader('Cache-Control', 'no-store');
-    res.setHeader('Content-Disposition', buildInlineFilename(source.source_key));
-
-    // Each WAV chunk is a self-contained file with its own 44-byte RIFF header
-    // whose declared length only covers that chunk. Streaming them back-to-back
-    // produces a file most players stop reading after the first chunk. Rebuild
-    // one continuous WAV: a single header sized for the combined PCM, followed
-    // by the raw sample data from every chunk. Non-WAV containers (e.g. WebM)
-    // concatenate into a valid stream as-is.
-    const wavFormat = /wav/i.test(mimeType)
-      ? await readCanonicalWavHeader(readableChunks[0].file_path)
-      : null;
-
-    if (wavFormat) {
-      let totalPcmBytes = 0;
-      for (const chunk of readableChunks) {
-        const { size } = await fs.promises.stat(chunk.file_path);
-        totalPcmBytes += Math.max(0, size - WAV_HEADER_BYTES);
-      }
-      res.write(buildWavHeader(wavFormat, totalPcmBytes));
-      for (const chunk of readableChunks) {
-        const stream = fs.createReadStream(chunk.file_path, { start: WAV_HEADER_BYTES });
-        stream.pipe(res, { end: false });
-        await finished(stream);
-      }
-      res.end();
-      return;
-    }
-
-    for (const chunk of readableChunks) {
-      const stream = fs.createReadStream(chunk.file_path);
-      stream.pipe(res, { end: false });
-      await finished(stream);
-    }
-
-    res.end();
-  } catch (err) {
-    res.status(500).json({ error: sanitizeError(err) });
-  }
-});
-
-router.post('/', (req, res) => {
-  try {
-    const manager = getRecordingManager(req);
-    const session = manager.createSession(req.session.userId, req.body || {});
-    res.status(201).json({ session });
-  } catch (err) {
-    respondWithMappedError(res, err, [
-      { pattern: /source|title|required|duplicate/i, status: 400 },
-    ]);
-  }
-});
-
-router.post('/:sessionId/chunks', async (req, res) => {
-  try {
-    const manager = getRecordingManager(req);
-    const body = await readChunkBody(req);
-    const result = manager.appendChunk(req.session.userId, req.params.sessionId, getChunkMetadata(req), body);
-    res.status(result.duplicate ? 200 : 201).json(result);
-  } catch (err) {
-    console.error('[Recordings] Chunk upload failed:', err);
-    respondWithMappedError(res, err, [
-      { pattern: /not found/i, status: 404 },
-      { pattern: /empty|required|unknown|non-negative|accepting|sequence|contiguous/i, status: 400 },
-    ]);
-  }
-});
-
-router.post('/:sessionId/finalize', (req, res) => {
-  try {
-    const manager = getRecordingManager(req);
-    const session = manager.finalizeSession(req.session.userId, req.params.sessionId, req.body || {});
-    res.json({ session });
-  } catch (err) {
-    respondWithMappedError(res, err, [
-      { pattern: /not found/i, status: 404 },
-    ]);
-  }
-});
-
-router.post('/:sessionId/retry', async (req, res) => {
-  try {
-    const manager = getRecordingManager(req);
-    const session = await manager.retrySession(req.session.userId, req.params.sessionId);
-    res.json({ session });
-  } catch (err) {
-    respondWithMappedError(res, err, [
-      { pattern: /not found/i, status: 404 },
-      { pattern: /configured/i, status: 400 },
-    ]);
-  }
-});
-
-router.delete('/:sessionId/segments/:segmentId', (req, res) => {
-  try {
-    const manager = getRecordingManager(req);
-    const session = manager.deleteTranscriptSegment(
-      req.session.userId,
-      req.params.sessionId,
-      req.params.segmentId,
-    );
-    res.json({ session });
-  } catch (err) {
-    respondWithMappedError(res, err, [
-      { pattern: /not found/i, status: 404 },
-      { pattern: /positive integer/i, status: 400 },
-    ]);
-  }
-});
-
-router.delete('/:sessionId', (req, res) => {
-  try {
-    const manager = getRecordingManager(req);
-    manager.deleteSession(req.session.userId, req.params.sessionId);
-    res.status(204).send();
-  } catch (err) {
-    respondWithMappedError(res, err, [
-      { pattern: /not found/i, status: 404 },
-    ]);
-  }
-});
-
-module.exports = router;

package/server/routes/screenHistory.js

@@ -1,190 +0,0 @@
-'use strict';
-
-const express = require('express');
-const db = require('../db/database');
-const { buildFtsQuery } = require('../db/ftsQuery');
-const { requireAuth } = require('../middleware/auth');
-const { getErrorMessage } = require('../services/bootstrap_helpers');
-
-const router = express.Router();
-
-router.use(requireAuth);
-
-function getTimelineService(req) {
-  return req.app?.locals?.timelineService || null;
-}
-
-function normalizeEntries(rawEntries) {
-  if (!Array.isArray(rawEntries)) {
-    throw new Error('entries must be an array.');
-  }
-  return rawEntries.map((entry) => {
-    if (!entry || typeof entry !== 'object') {
-      throw new Error('Each entry must be an object.');
-    }
-    const text = String(entry.text || '').replace(/\s+/g, ' ').trim();
-    if (!text) {
-      throw new Error('Entry text is required.');
-    }
-    return {
-      capturedAt: entry.capturedAt,
-      frontmostApp: entry.frontmostApp,
-      windowTitle: entry.windowTitle,
-      text,
-      ocrConfidence: entry.ocrConfidence,
-    };
-  });
-}
-
-router.get('/search', (req, res) => {
-  const { q, limit = 50, offset = 0 } = req.query;
-  const userId = req.session.userId;
-
-  try {
-    let results = [];
-    const ftsQuery = q ? buildFtsQuery(q) : null;
-    if (ftsQuery) {
-      // Full text search. buildFtsQuery sanitizes user input so FTS5 operator
-      // characters (hyphens, AND/OR/NOT) don't throw and 500 the request.
-      results = db.prepare(`
-        SELECT
-          s.id,
-          s.timestamp,
-          s.captured_at,
-          s.captured_at AS capturedAt,
-          s.device_id,
-          s.device_id AS deviceId,
-          s.device_label,
-          s.device_label AS deviceLabel,
-          s.app_name,
-          s.app_name AS appName,
-          s.window_title,
-          s.window_title AS windowTitle,
-          s.text_content,
-          s.ocr_engine,
-          s.ocr_engine AS ocrEngine,
-          s.ocr_confidence,
-          s.ocr_confidence AS ocrConfidence
-        FROM screen_history_fts fts
-        JOIN screen_history s ON fts.rowid = s.id
-        WHERE screen_history_fts MATCH ? AND s.user_id = ?
-        ORDER BY s.timestamp DESC
-        LIMIT ? OFFSET ?
-      `).all(ftsQuery, userId, Number(limit), Number(offset));
-    } else if (q) {
-      // Query had no usable search tokens — return no matches rather than error.
-      results = [];
-    } else {
-      // Recent history
-      results = db.prepare(`
-        SELECT
-          id,
-          timestamp,
-          captured_at,
-          captured_at AS capturedAt,
-          device_id,
-          device_id AS deviceId,
-          device_label,
-          device_label AS deviceLabel,
-          app_name,
-          app_name AS appName,
-          window_title,
-          window_title AS windowTitle,
-          text_content,
-          ocr_engine,
-          ocr_engine AS ocrEngine,
-          ocr_confidence,
-          ocr_confidence AS ocrConfidence
-        FROM screen_history
-        WHERE user_id = ?
-        ORDER BY timestamp DESC
-        LIMIT ? OFFSET ?
-      `).all(userId, Number(limit), Number(offset));
-    }
-
-    res.json({ results });
-  } catch (err) {
-    console.error('[ScreenHistory] Search error:', getErrorMessage(err));
-    res.status(500).json({ error: 'Failed to search screen history' });
-  }
-});
-
-router.post('/entries', express.json(), (req, res) => {
-  try {
-    const userId = req.session.userId;
-    const deviceId = String(req.body?.deviceId || '').trim();
-    const activationId = String(req.body?.activationId || '').trim();
-    if (!deviceId) {
-      return res.status(400).json({ error: 'deviceId is required.' });
-    }
-    if (!activationId) {
-      return res.status(400).json({ error: 'activationId is required.' });
-    }
-
-    const device = db.prepare(
-      `SELECT device_id, label, activation_id, revoked_at, passive_history_enabled
-       FROM desktop_companion_devices
-       WHERE user_id = ? AND device_id = ?
-       LIMIT 1`
-    ).get(userId, deviceId);
-    if (!device) {
-      return res.status(404).json({ error: 'Desktop companion device not found.' });
-    }
-    if (device.revoked_at) {
-      return res.status(403).json({ error: 'Desktop companion device has been revoked.' });
-    }
-    if (String(device.activation_id || '') !== activationId) {
-      return res.status(403).json({ error: 'activationId does not match the registered desktop companion device.' });
-    }
-    if (device.passive_history_enabled !== 1) {
-      return res.status(403).json({ error: 'Passive screen history is disabled for this desktop companion device.' });
-    }
-
-    const entries = normalizeEntries(req.body?.entries);
-    const timelineService = getTimelineService(req);
-    if (!timelineService || typeof timelineService.storeScreenEntries !== 'function') {
-      return res.status(503).json({ error: 'Timeline service is unavailable.' });
-    }
-
-    const result = timelineService.storeScreenEntries({
-      userId,
-      deviceId,
-      deviceLabel: device.label || deviceId,
-      entries,
-    });
-    req.app?.locals?.desktopCompanionRegistry?.updatePassiveHistoryState?.(
-      userId,
-      deviceId,
-      {
-        enabled: true,
-        lastUploadedAt: new Date().toISOString(),
-        lastError: null,
-      },
-    );
-    res.status(201).json({
-      ok: true,
-      insertedCount: result.insertedCount,
-    });
-  } catch (err) {
-    const message = getErrorMessage(err);
-    const userId = req.session?.userId;
-    const deviceId = String(req.body?.deviceId || '').trim();
-    if (userId && deviceId) {
-      try {
-        req.app?.locals?.desktopCompanionRegistry?.updatePassiveHistoryState?.(
-          userId,
-          deviceId,
-          {
-            enabled: true,
-            lastError: message,
-          },
-        );
-      } catch {
-        // Best-effort; do not mask the original ingest failure.
-      }
-    }
-    res.status(400).json({ error: message });
-  }
-});
-
-module.exports = router;