neoagent 2.3.1-beta.87 → 2.3.1-beta.89

package/server/services/browser/controller.js

@@ -490,7 +490,9 @@ class BrowserController {
         url: currentUrl,
         status: response?.status() || 0,
         bodyText,
-        screenshotPath: screenshot?.screenshotPath || null
+        screenshotPath: screenshot?.screenshotPath || null,
+        artifactId: screenshot?.artifactId || null,
+        fullPath: screenshot?.fullPath || null
       };
     } catch (err) {
       let screenshot = null;
@@ -498,7 +500,9 @@ class BrowserController {
       return {
         error: err.message,
         url,
-        screenshotPath: screenshot?.screenshotPath || null
+        screenshotPath: screenshot?.screenshotPath || null,
+        artifactId: screenshot?.artifactId || null,
+        fullPath: screenshot?.fullPath || null
       };
     }
   }
@@ -540,7 +544,9 @@ class BrowserController {
         success: true,
         url: page.url(),
         title: await page.title(),
-        screenshotPath: screenshotResult?.screenshotPath || null
+        screenshotPath: screenshotResult?.screenshotPath || null,
+        artifactId: screenshotResult?.artifactId || null,
+        fullPath: screenshotResult?.fullPath || null
       };
     } catch (err) {
       return { error: err.message };
@@ -569,7 +575,9 @@ class BrowserController {
         y: py,
         url: page.url(),
         title: await page.title(),
-        screenshotPath: screenshotResult?.screenshotPath || null
+        screenshotPath: screenshotResult?.screenshotPath || null,
+        artifactId: screenshotResult?.artifactId || null,
+        fullPath: screenshotResult?.fullPath || null
       };
     } catch (err) {
       return { error: err.message };
@@ -593,7 +601,9 @@ class BrowserController {
         success: true,
         url: page.url(),
         title: await page.title(),
-        screenshotPath: screenshotResult?.screenshotPath || null
+        screenshotPath: screenshotResult?.screenshotPath || null,
+        artifactId: screenshotResult?.artifactId || null,
+        fullPath: screenshotResult?.fullPath || null
       };
     } catch (err) {
       return { error: err.message };
@@ -624,7 +634,9 @@ class BrowserController {
       return {
         success: true,
         typed: text,
-        screenshotPath: screenshotResult?.screenshotPath || null
+        screenshotPath: screenshotResult?.screenshotPath || null,
+        artifactId: screenshotResult?.artifactId || null,
+        fullPath: screenshotResult?.fullPath || null
       };
     } catch (err) {
       return { error: err.message };
@@ -650,7 +662,9 @@ class BrowserController {
       return {
         success: true,
         typed: String(text || ''),
-        screenshotPath: screenshotResult?.screenshotPath || null
+        screenshotPath: screenshotResult?.screenshotPath || null,
+        artifactId: screenshotResult?.artifactId || null,
+        fullPath: screenshotResult?.fullPath || null
       };
     } catch (err) {
       return { error: err.message };
@@ -674,7 +688,9 @@ class BrowserController {
       return {
         success: true,
         key: normalized,
-        screenshotPath: screenshotResult?.screenshotPath || null
+        screenshotPath: screenshotResult?.screenshotPath || null,
+        artifactId: screenshotResult?.artifactId || null,
+        fullPath: screenshotResult?.fullPath || null
       };
     } catch (err) {
       return { error: err.message };

package/server/services/runtime/backends/local-vm.js

@@ -165,13 +165,45 @@ class VmBrowserProvider {
   }
 
   async #materialize(result) {
-    if (!result || !result.fullPath || !this.artifactStore || this.userId == null) {
+    if (!result || !this.artifactStore || this.userId == null) {
       return result;
     }
-    const file = await this.client.request('POST', '/files/read', {
-      path: result.fullPath,
-      encoding: 'base64',
-    });
+
+    const readablePathCandidates = [];
+    if (result.fullPath) {
+      readablePathCandidates.push(String(result.fullPath));
+    }
+    if (typeof result.screenshotPath === 'string' && result.screenshotPath.startsWith('/screenshots/')) {
+      readablePathCandidates.push(result.screenshotPath);
+    }
+    if (readablePathCandidates.length === 0) {
+      return result;
+    }
+
+    let file = null;
+    for (const candidate of readablePathCandidates) {
+      try {
+        file = await this.client.request('POST', '/files/read', {
+          path: candidate,
+          encoding: 'base64',
+        });
+        if (file?.content) {
+          break;
+        }
+      } catch {}
+    }
+    if (!file?.content) {
+      if (typeof result.screenshotPath === 'string' && result.screenshotPath.startsWith('/screenshots/')) {
+        return {
+          ...result,
+          screenshotPath: null,
+          artifactId: result.artifactId || null,
+          fullPath: result.fullPath || null,
+        };
+      }
+      return result;
+    }
+
     const allocation = this.artifactStore.allocateFile(this.userId, {
       kind: 'browser-screenshot',
       backend: 'vm',

package/server/services/social_video/service.js

@@ -5,15 +5,17 @@ const fsp = require('fs/promises');
 const path = require('path');
 const { randomUUID } = require('crypto');
 
+const db = require('../../db/database');
 const { DATA_DIR } = require('../../../runtime/paths');
 const { CLIExecutor } = require('../cli/executor');
-const { isDeepgramConfigured, transcribeChunkWithDeepgram } = require('../recordings/deepgram');
 const { getAdapterForPlatform } = require('./adapters');
 const { decideTranscriptPath, parseCaptionText, pickCaptionTrack } = require('./captions');
 const { inferImageContentType, pickDeterministicFrameSecond } = require('./frame');
 const { extractPublicMetadataFromHtml } = require('./metadata');
 const { shapeSocialVideoResult } = require('./result');
 const { normalizeAndDetectPlatform } = require('./url');
+const { isMainAgent } = require('../agents/manager');
+const { resolveSttModel, transcribeVoiceInput } = require('../voice/providers');
 
 const SOCIAL_VIDEO_TMP_DIR = path.join(DATA_DIR, 'social-video-temp');
 fs.mkdirSync(SOCIAL_VIDEO_TMP_DIR, { recursive: true });
@@ -70,6 +72,60 @@ function unwrapBrowserExtractValue(payload) {
   return '';
 }
 
+function parseStoredSettingValue(value) {
+  if (typeof value !== 'string') {
+    return value;
+  }
+  try {
+    return JSON.parse(value);
+  } catch {
+    return value;
+  }
+}
+
+function readStoredSetting(userId, agentId, key) {
+  if (!userId) {
+    return null;
+  }
+
+  if (agentId) {
+    const agentRow = db.prepare(
+      'SELECT value FROM agent_settings WHERE user_id = ? AND agent_id = ? AND key = ?',
+    ).get(userId, agentId, key);
+    if (agentRow) {
+      return parseStoredSettingValue(agentRow.value);
+    }
+  }
+
+  if (!agentId || isMainAgent(userId, agentId)) {
+    const userRow = db.prepare(
+      'SELECT value FROM user_settings WHERE user_id = ? AND key = ?',
+    ).get(userId, key);
+    if (userRow) {
+      return parseStoredSettingValue(userRow.value);
+    }
+  }
+
+  return null;
+}
+
+function resolveVoiceSttConfigFromSettings(settings = {}) {
+  const provider = String(
+    settings.voice_stt_provider
+    || settings.default_recording_transcription_provider
+    || '',
+  ).trim().toLowerCase() || 'openai';
+  const model = String(
+    settings.voice_stt_model
+    || settings.default_recording_transcription_model
+    || '',
+  ).trim();
+  return {
+    provider,
+    model: resolveSttModel(provider, model),
+  };
+}
+
 function fileExists(filePath) {
   try {
     return fs.statSync(filePath).isFile();
@@ -124,6 +180,8 @@ class SocialVideoService {
     this.artifactStore = options.artifactStore || null;
     this.runtimeManager = options.runtimeManager || null;
     this.cliExecutor = options.cliExecutor || new CLIExecutor();
+    this.voiceTranscriber = options.voiceTranscriber || transcribeVoiceInput;
+    this.voiceSettingsResolver = options.voiceSettingsResolver || ((userId, agentId) => this.#resolveVoiceSttConfig(userId, agentId));
     this.ytDlpBin = String(process.env.YT_DLP_BIN || 'yt-dlp').trim() || 'yt-dlp';
     this.ffmpegBin = String(process.env.FFMPEG_BIN || 'ffmpeg').trim() || 'ffmpeg';
     this._healthCache = {
@@ -148,10 +206,7 @@ class SocialVideoService {
       ready: ytDlp.available && ffmpeg.available,
       dependencies: [ytDlp, ffmpeg],
       speechToText: {
-        configured: isDeepgramConfigured(),
-        note: isDeepgramConfigured()
-          ? 'Deepgram is configured for speech-to-text fallback.'
-          : 'DEEPGRAM_API_KEY is not configured. Extraction still works when platform captions are available.',
+        note: 'Transcript fallback uses the configured voice STT provider from Flutter settings.',
       },
       checkedAt: new Date().toISOString(),
     };
@@ -167,6 +222,7 @@ class SocialVideoService {
     const warnings = [];
     const errors = [];
     const source = String(sourceUrl || '').trim();
+    const agentId = options.agentId || null;
     let jobDir = null;
 
     try {
@@ -208,6 +264,8 @@ class SocialVideoService {
         captionTrack,
         transcriptDecision,
         jobDir,
+        userId,
+        agentId,
         warnings,
       });
 
@@ -375,14 +433,14 @@ class SocialVideoService {
   }
 
   async #readMediaInfo(normalizedUrl, jobDir) {
-    const infoPath = path.join(jobDir, 'media-info.json');
-    const command = `${shellEscape(this.ytDlpBin)} --no-playlist --skip-download --dump-single-json -- ${shellEscape(normalizedUrl)}`;
-    const result = await this.#runCommand(command, { cwd: jobDir, timeout: 4 * 60 * 1000 });
-    const raw = String(result.stdout || '').trim();
-    if (!raw) {
-      throw new Error('yt-dlp returned empty media metadata output.');
+    const infoTemplate = path.join(jobDir, 'media.%(ext)s');
+    const infoPath = path.join(jobDir, 'media.info.json');
+    const command = `${shellEscape(this.ytDlpBin)} --quiet --no-warnings --no-playlist --skip-download --write-info-json --no-clean-infojson -o ${shellEscape(infoTemplate)} -- ${shellEscape(normalizedUrl)}`;
+    await this.#runCommand(command, { cwd: jobDir, timeout: 4 * 60 * 1000 });
+    if (!fileExists(infoPath)) {
+      throw new Error('yt-dlp did not produce an info JSON artifact.');
     }
-    await fsp.writeFile(infoPath, `${raw}\n`, 'utf8');
+    const raw = String(await fsp.readFile(infoPath, 'utf8')).trim();
     let parsed;
     try {
       parsed = JSON.parse(raw);
@@ -407,15 +465,10 @@ class SocialVideoService {
       context.warnings.push('Caption track was present but transcript text was empty. Falling back to speech-to-text.');
     }
 
-    if (!isDeepgramConfigured()) {
-      context.warnings.push('Captions unavailable and DEEPGRAM_API_KEY is not configured; transcript could not be generated.');
-      return {
-        text: '',
-        source: 'unavailable',
-      };
-    }
-
-    const transcript = await this.#transcribeViaStt(context.sourceUrl, context.jobDir);
+    const transcript = await this.#transcribeViaStt(context).catch((error) => {
+      context.warnings.push(`Speech-to-text fallback failed: ${error.message}`);
+      return '';
+    });
     return {
       text: transcript,
       source: transcript ? 'stt' : 'unavailable',
@@ -431,23 +484,41 @@ class SocialVideoService {
     return parseCaptionText(raw, captionTrack.ext);
   }
 
-  async #transcribeViaStt(sourceUrl, jobDir) {
-    const template = path.join(jobDir, 'audio.%(ext)s');
-    const command = `${shellEscape(this.ytDlpBin)} --no-playlist -f bestaudio -- ${shellEscape(sourceUrl)} -o ${shellEscape(template)}`;
-    await this.#runCommand(command, { cwd: jobDir, timeout: 10 * 60 * 1000 });
+  async #transcribeViaStt(context) {
+    const template = path.join(context.jobDir, 'audio.%(ext)s');
+    const command = `${shellEscape(this.ytDlpBin)} --quiet --no-warnings --no-playlist -o ${shellEscape(template)} -f bestaudio -- ${shellEscape(context.sourceUrl)}`;
+    await this.#runCommand(command, { cwd: context.jobDir, timeout: 10 * 60 * 1000 });
 
-    const audioPath = firstFileMatching(jobDir, 'audio.');
+    const audioPath = firstFileMatching(context.jobDir, 'audio.');
     if (!audioPath || !fileExists(audioPath)) {
       throw new Error('Audio download succeeded but no audio file was created.');
     }
 
-    const audioBytes = await fsp.readFile(audioPath);
-    const deepgramResult = await transcribeChunkWithDeepgram({
-      audioBytes,
+    const sttConfig = await Promise.resolve(
+      this.voiceSettingsResolver(context.userId, context.agentId),
+    );
+    return this.voiceTranscriber(audioPath, {
+      provider: sttConfig?.provider || 'openai',
+      model: sttConfig?.model || '',
       mimeType: detectMimeFromFile(audioPath),
     });
-    const transcript = deepgramResult?.results?.channels?.[0]?.alternatives?.[0]?.transcript;
-    return String(transcript || '').trim();
+  }
+
+  async #resolveVoiceSttConfig(userId, agentId) {
+    return resolveVoiceSttConfigFromSettings({
+      voice_stt_provider: readStoredSetting(userId, agentId, 'voice_stt_provider'),
+      voice_stt_model: readStoredSetting(userId, agentId, 'voice_stt_model'),
+      default_recording_transcription_provider: readStoredSetting(
+        userId,
+        agentId,
+        'default_recording_transcription_provider',
+      ),
+      default_recording_transcription_model: readStoredSetting(
+        userId,
+        agentId,
+        'default_recording_transcription_model',
+      ),
+    });
   }
 
   async #resolveFrameImage(context) {
@@ -469,7 +540,7 @@ class SocialVideoService {
 
   async #extractFrameFromVideo(context) {
     const template = path.join(context.jobDir, 'video.%(ext)s');
-    const downloadCommand = `${shellEscape(this.ytDlpBin)} --no-playlist -f "bv*[ext=mp4]+ba[ext=m4a]/b[ext=mp4]/best" --merge-output-format mp4 -- ${shellEscape(context.sourceUrl)} -o ${shellEscape(template)}`;
+    const downloadCommand = `${shellEscape(this.ytDlpBin)} --quiet --no-warnings --no-playlist -o ${shellEscape(template)} -f "bv*[ext=mp4]+ba[ext=m4a]/b[ext=mp4]/best" --merge-output-format mp4 -- ${shellEscape(context.sourceUrl)}`;
     await this.#runCommand(downloadCommand, { cwd: context.jobDir, timeout: 14 * 60 * 1000 });
 
     const videoPath = firstFileMatching(context.jobDir, 'video.');
@@ -479,7 +550,7 @@ class SocialVideoService {
 
     const framePath = path.join(context.jobDir, 'frame.jpg');
     const frameSecond = pickDeterministicFrameSecond(context.mediaInfo.duration);
-    const frameCommand = `${shellEscape(this.ffmpegBin)} -y -hide_banner -loglevel error -ss ${frameSecond} -i ${shellEscape(videoPath)} -frames:v 1 -q:v 2 ${shellEscape(framePath)}`;
+    const frameCommand = `${shellEscape(this.ffmpegBin)} -hwaccel none -y -hide_banner -loglevel error -ss ${frameSecond} -i ${shellEscape(videoPath)} -frames:v 1 -q:v 2 ${shellEscape(framePath)}`;
     await this.#runCommand(frameCommand, { cwd: context.jobDir, timeout: 2 * 60 * 1000 });
 
     if (!fileExists(framePath)) {
@@ -505,7 +576,7 @@ class SocialVideoService {
         source: 'thumbnail',
       };
     }
-    const allocation = this.artifactStore.allocateFile(userId, {
+    const allocation = await Promise.resolve(this.artifactStore.allocateFile(userId, {
       kind: 'social-video-frame',
       extension: guessedExtension,
       contentType: mimeType,
@@ -513,9 +584,11 @@ class SocialVideoService {
       metadata: {
         source: 'social-video-thumbnail',
       },
-    });
+    }));
     await fsp.writeFile(allocation.storagePath, buffer);
-    const finalized = this.artifactStore.finalizeFile(allocation.artifactId, allocation.storagePath);
+    const finalized = await Promise.resolve(
+      this.artifactStore.finalizeFile(allocation.artifactId, allocation.storagePath),
+    );
     return {
       url: finalized.url,
       artifactId: finalized.artifactId,
@@ -572,5 +645,6 @@ module.exports = {
   firstFileMatching,
   pickBestThumbnail,
   classifyExtractionError,
+  resolveVoiceSttConfigFromSettings,
   shellEscape,
 };