@thevinci/web 1.0.0

package/server/lib/tts/base-url.js

@@ -0,0 +1,62 @@
+const LOCAL_BASE_URL_HOSTS = new Set([
+  'localhost',
+  '127.0.0.1',
+  '::1',
+  'host.docker.internal',
+]);
+
+const isEnvFlagEnabled = (value) => {
+  if (value === true || value === 1) return true;
+  if (typeof value !== 'string') return false;
+  const normalized = value.trim().toLowerCase();
+  return normalized === '1' || normalized === 'true';
+};
+
+const normalizeHostname = (hostname) => {
+  if (typeof hostname !== 'string') return '';
+  const trimmed = hostname.trim().toLowerCase();
+  if (!trimmed) return '';
+  if (trimmed.startsWith('[') && trimmed.endsWith(']')) {
+    return trimmed.slice(1, -1);
+  }
+  return trimmed;
+};
+
+const isAllowedLocalHost = (hostname) => {
+  const normalized = normalizeHostname(hostname);
+  return LOCAL_BASE_URL_HOSTS.has(normalized);
+};
+
+export const normalizeCustomOpenAIBaseURL = (value) => {
+  if (typeof value !== 'string' || !value.trim()) {
+    return { value: undefined };
+  }
+
+  let parsed;
+  try {
+    parsed = new URL(value.trim());
+  } catch {
+    return { error: 'Custom server URL is invalid' };
+  }
+
+  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
+    return { error: 'Custom server URL must use http or https' };
+  }
+
+  if (parsed.username || parsed.password) {
+    return { error: 'Custom server URL must not include credentials' };
+  }
+
+  const allowRemote = isEnvFlagEnabled(process.env.VINCI_ALLOW_REMOTE_OPENAI_COMPAT_URLS);
+  if (!allowRemote && !isAllowedLocalHost(parsed.hostname)) {
+    return {
+      error: 'Remote custom server URLs are disabled. Set VINCI_ALLOW_REMOTE_OPENAI_COMPAT_URLS=true to allow this host.',
+    };
+  }
+
+  parsed.hash = '';
+  parsed.search = '';
+  const pathname = parsed.pathname.replace(/\/+$/, '');
+  const normalizedPath = pathname.length > 0 ? pathname : '';
+  return { value: `${parsed.protocol}//${parsed.host}${normalizedPath}` };
+};

package/server/lib/tts/capability-runtime.js

@@ -0,0 +1,31 @@
+export const detectSayTtsCapability = async (processLike) => {
+  let sayTTSCapability = { available: false, voices: [], reason: 'Not checked' };
+
+  if (processLike.platform === 'darwin') {
+    try {
+      const { exec } = await import('child_process');
+      const { promisify } = await import('util');
+      const execAsync = promisify(exec);
+      const { stdout } = await execAsync('say -v "?"');
+      const voices = stdout.split('\n')
+        .filter((line) => line.trim())
+        .map((line) => {
+          const match = line.match(/^(.+?)\s+([a-zA-Z]{2}_[a-zA-Z]{2,3})\s+#/);
+          if (match) {
+            return { name: match[1].trim(), locale: match[2] };
+          }
+          return null;
+        })
+        .filter(Boolean);
+      sayTTSCapability = { available: true, voices };
+      console.log(`macOS Say TTS available with ${voices.length} voices`);
+    } catch (error) {
+      sayTTSCapability = { available: false, voices: [], reason: 'say command not available' };
+      console.log('macOS Say TTS not available:', error.message);
+    }
+  } else {
+    sayTTSCapability = { available: false, voices: [], reason: 'Not macOS' };
+  }
+
+  return sayTTSCapability;
+};

package/server/lib/tts/index.js

@@ -0,0 +1,19 @@
+/**
+ * TTS Module Entry Point
+ *
+ * Public export surface for the Text-to-Speech domain module.
+ */
+
+export {
+  ttsService,
+  TTSService,
+  TTS_VOICES,
+} from './service.js';
+
+export {
+  summarizeText,
+  sanitizeForTTS,
+  sanitizeForNote,
+} from '../text/summarization.js';
+
+export { transcribeAudio } from './stt.js';

package/server/lib/tts/routes.js

@@ -0,0 +1,261 @@
+import express from 'express';
+import { normalizeCustomOpenAIBaseURL } from './base-url.js';
+import { summarizeText, sanitizeForTTS, sanitizeForNote } from '../text/summarization.js';
+
+export function registerTtsRoutes(app, { sayTTSCapability }) {
+  let ttsModulePromise = null;
+  const getTtsModule = async () => {
+    if (!ttsModulePromise) {
+      ttsModulePromise = import('./index.js');
+    }
+    return ttsModulePromise;
+  };
+
+  app.post('/api/voice/token', async (req, res) => {
+    console.log('[Voice] Token request received:', {
+      contentType: req.headers['content-type'] || null,
+    });
+    try {
+      const openaiApiKey = process.env.OPENAI_API_KEY;
+      console.log('[Voice] OpenAI API Key present:', !!openaiApiKey);
+
+      if (!openaiApiKey) {
+        return res.status(503).json({
+          allowed: false,
+          error: 'OpenAI voice service not configured. Set OPENAI_API_KEY environment variable.'
+        });
+      }
+
+      // Return success - OpenAI TTS is available
+      res.json({
+        allowed: true,
+        provider: 'openai',
+        message: 'OpenAI TTS is available'
+      });
+    } catch (error) {
+      console.error('[Voice] Token generation error:', error);
+      res.status(500).json({
+        allowed: false,
+        error: 'Voice service error'
+      });
+    }
+  });
+
+  // Server-side TTS endpoint - streams audio from OpenAI TTS API
+  app.post('/api/tts/speak', async (req, res) => {
+    try {
+      const { text, voice = 'nova', model = 'gpt-4o-mini-tts', speed = 0.9, instructions, providerId, modelId, apiKey, baseURL } = req.body || {};
+
+      const normalizedBaseURLResult = normalizeCustomOpenAIBaseURL(baseURL);
+      if (normalizedBaseURLResult.error) {
+        return res.status(400).json({ error: normalizedBaseURLResult.error });
+      }
+      const normalizedBaseURL = normalizedBaseURLResult.value;
+
+      console.log('[TTS] Request received:', { voice, model, speed, textLength: text?.length, hasApiKey: !!apiKey, hasBaseURL: !!baseURL });
+
+      if (!text || typeof text !== 'string' || !text.trim()) {
+        return res.status(400).json({ error: 'Text is required' });
+      }
+
+      // Dynamically import the TTS service (ESM)
+      const { ttsService } = await getTtsModule();
+
+      // Check availability - server-configured key, client-provided key, or custom server URL
+      const hasServerKey = ttsService.isAvailable();
+      const hasClientKey = apiKey && typeof apiKey === 'string' && apiKey.trim().length > 0;
+      const hasCustomBaseURL = typeof normalizedBaseURL === 'string' && normalizedBaseURL.length > 0;
+      
+      if (!hasServerKey && !hasClientKey && !hasCustomBaseURL) {
+        return res.status(503).json({ 
+          error: 'TTS service not available. Please configure OpenAI in OpenCode, provide an API key, or set a custom server URL in settings.' 
+        });
+      }
+
+      let textToSpeak = text.trim();
+
+      // Historical summarize request fields are intentionally ignored. The
+      // model-backed summarization provider is retired.
+
+      const result = await ttsService.generateSpeechStream({
+        text: textToSpeak,
+        voice,
+        model,
+        speed,
+        instructions,
+        apiKey: hasClientKey ? apiKey.trim() : undefined,
+        baseURL: hasCustomBaseURL ? normalizedBaseURL : undefined,
+      });
+
+      res.setHeader('Content-Type', result.contentType);
+      res.setHeader('Cache-Control', 'no-cache');
+      res.setHeader('Content-Length', result.buffer.length);
+      res.send(result.buffer);
+      } catch (error) {
+        console.error('[TTS] Error:', error);
+        if (!res.headersSent) {
+          const { model: m, voice: v, baseURL: b } = req.body || {};
+          res.status(500).json({ 
+            error: error instanceof Error ? error.message : 'TTS generation failed',
+            detail: { model: m, voice: v, hasBaseURL: !!b },
+          });
+        }
+      }
+  });
+
+  app.post('/api/text/summarize', async (req, res) => {
+    try {
+      const { text, threshold = 200, maxLength = 500, mode } = req.body || {};
+
+      if (!text || typeof text !== 'string' || !text.trim()) {
+        return res.status(400).json({ error: 'Text is required' });
+      }
+
+      const result = await summarizeText({
+        text,
+        threshold,
+        maxLength,
+        mode: typeof mode === 'string' ? mode : 'tts',
+      });
+
+      return res.json(result);
+    } catch (error) {
+      console.error('[Summarize] Error:', error);
+      const sanitized = typeof req.body?.mode === 'string' && req.body.mode === 'note'
+        ? sanitizeForNote(req.body?.text || '')
+        : sanitizeForTTS(req.body?.text || '');
+      return res.json({ summary: sanitized, summarized: false, reason: error.message });
+    }
+  });
+
+       
+  // TTS status endpoint
+  app.get('/api/tts/status', async (_req, res) => {
+    try {
+      const { ttsService } = await getTtsModule();
+      res.json({
+        available: ttsService.isAvailable(),
+        voices: [
+          'alloy', 'ash', 'ballad', 'coral', 'echo', 'fable',
+          'nova', 'onyx', 'sage', 'shimmer', 'verse', 'marin', 'cedar'
+        ]
+      });
+    } catch (error) {
+      res.status(500).json({ error: 'Failed to check TTS status' });
+    }
+  });
+
+  // macOS 'say' command TTS status endpoint - returns cached capability from startup
+  app.get('/api/tts/say/status', (_req, res) => {
+    res.json(sayTTSCapability);
+  });
+
+  // macOS 'say' command TTS speak endpoint
+  app.post('/api/tts/say/speak', async (req, res) => {
+    try {
+      const { text, voice = 'Samantha', rate = 200 } = req.body || {};
+      
+      if (!text || typeof text !== 'string' || !text.trim()) {
+        return res.status(400).json({ error: 'Text is required' });
+      }
+      
+      // Check if we're on macOS
+      if (process.platform !== 'darwin') {
+        return res.status(503).json({ error: 'macOS say command not available on this platform' });
+      }
+      
+      const { exec } = await import('child_process');
+      const { promisify } = await import('util');
+      const fs = await import('fs');
+      const os = await import('os');
+      const path = await import('path');
+      const execAsync = promisify(exec);
+      
+      // Create temp file for audio output (use m4a for browser compatibility)
+      const tempDir = os.tmpdir();
+      const tempFile = path.join(tempDir, `say-${Date.now()}.m4a`);
+      
+      // Escape text for shell - escape both single quotes and double quotes
+      const escapedText = text.trim().replace(/'/g, "'\\''").replace(/"/g, '\\"');
+      
+      // Generate audio file using 'say' command
+      // -o outputs to file, -r sets rate (words per minute)
+      // --data-format=aac outputs as m4a which browsers can decode
+      const cmd = `say -v "${voice}" -r ${rate} -o "${tempFile}" --data-format=aac '${escapedText}'`;
+      console.log('[TTS-Say] Generating speech:', { textLength: text.length, voice, rate });
+      
+      await execAsync(cmd);
+      
+      // Read the generated audio file
+      const audioBuffer = await fs.promises.readFile(tempFile);
+      
+      // Clean up temp file
+      fs.promises.unlink(tempFile).catch(() => {});
+      
+      // Send audio response
+      res.setHeader('Content-Type', 'audio/mp4');
+      res.setHeader('Content-Length', audioBuffer.length);
+      res.send(audioBuffer);
+      
+    } catch (error) {
+      console.error('[TTS-Say] Error:', error);
+      res.status(500).json({
+        error: error instanceof Error ? error.message : 'Say command failed'
+      });
+    }
+  });
+
+  // Server-side STT: receive raw audio, proxy to OpenAI-compatible transcription endpoint
+  app.post(
+    '/api/stt/transcribe',
+    express.raw({ type: (req) => (req.headers['content-type'] || '').startsWith('audio/'), limit: '20mb' }),
+    async (req, res) => {
+      try {
+        const { transcribeAudio } = await import('./stt.js');
+
+        const mimeType = (req.headers['content-type'] || 'audio/webm').split(',')[0].trim();
+        const baseURL = typeof req.headers['x-base-url'] === 'string' ? req.headers['x-base-url'].trim() : '';
+        const model = typeof req.headers['x-model'] === 'string' && req.headers['x-model'].trim().length > 0
+          ? req.headers['x-model'].trim()
+          : 'deepdml/faster-whisper-large-v3-turbo-ct2';
+        const language = typeof req.headers['x-language'] === 'string' && req.headers['x-language'].trim().length > 0
+          ? req.headers['x-language'].trim()
+          : undefined;
+
+        if (!req.body || !Buffer.isBuffer(req.body) || req.body.length === 0) {
+          return res.status(400).json({ error: 'Audio data is required' });
+        }
+
+        if (!baseURL) {
+          return res.status(400).json({ error: 'X-Base-URL header is required' });
+        }
+
+        console.log('[STT] Transcribing audio:', {
+          bytes: req.body.length,
+          mimeType,
+          model,
+          baseURL,
+          language,
+        });
+
+        const transcript = await transcribeAudio({
+          audioBuffer: req.body,
+          mimeType,
+          model,
+          baseURL,
+          language,
+        });
+
+        console.log('[STT] Transcript:', transcript?.slice(0, 120));
+        res.json({ transcript: transcript ?? '' });
+      } catch (error) {
+        console.error('[STT] Error:', error);
+        if (!res.headersSent) {
+          res.status(500).json({
+            error: error instanceof Error ? error.message : 'Transcription failed',
+          });
+        }
+      }
+    }
+  );
+}

package/server/lib/tts/routes.test.js

@@ -0,0 +1,53 @@
+import { describe, expect, it } from 'vitest';
+import express from 'express';
+import request from 'supertest';
+
+import { registerTtsRoutes } from './routes.js';
+
+const createApp = () => {
+  const app = express();
+  app.use(express.json());
+  registerTtsRoutes(app, {
+    resolveZenModel: async () => 'gpt-5-nano',
+    sayTTSCapability: null,
+  });
+  return app;
+};
+
+describe('tts routes', () => {
+  it('returns local note fallback while model summarization is retired', async () => {
+    const response = await request(createApp())
+      .post('/api/text/summarize')
+      .send({
+        text: 'First sentence. Second sentence with the useful insight.',
+        threshold: 0,
+        maxLength: 100,
+        mode: 'note',
+      });
+
+    expect(response.status).toBe(200);
+    expect(response.body).toMatchObject({
+      summary: 'First sentence.',
+      summarized: false,
+      reason: 'Model summarization provider unavailable',
+    });
+  });
+
+  it('keeps notification fallback behavior without calling zen', async () => {
+    const response = await request(createApp())
+      .post('/api/text/summarize')
+      .send({
+        text: 'Notification text that should fall back cleanly.',
+        threshold: 0,
+        maxLength: 100,
+        mode: 'notification',
+      });
+
+    expect(response.status).toBe(200);
+    expect(response.body).toMatchObject({
+      summary: 'Notification text that should fall back cleanly.',
+      summarized: false,
+      reason: 'Model summarization provider unavailable',
+    });
+  });
+});

package/server/lib/tts/service.js

@@ -0,0 +1,178 @@
+/**
+ * Server-side Text-to-Speech Service
+ *
+ * Uses OpenAI's TTS API to generate audio on the server and stream it to clients.
+ * This bypasses mobile Safari's audio context restrictions.
+ */
+
+import OpenAI from 'openai';
+import { readAuthFile } from '../opencode/auth.js';
+import { normalizeCustomOpenAIBaseURL } from './base-url.js';
+
+// Voice options from OpenAI
+export const TTS_VOICES = [
+  'alloy', 'ash', 'ballad', 'coral', 'echo', 'fable',
+  'nova', 'onyx', 'sage', 'shimmer', 'verse', 'marin', 'cedar'
+];
+
+function getOpenAIApiKey() {
+  // First check environment variable
+  const envKey = process.env.OPENAI_API_KEY;
+  if (envKey) {
+    return envKey;
+  }
+
+  // Then check opencode auth file (same as usage tracker)
+  try {
+    const auth = readAuthFile();
+    // Check for openai, codex, or chatgpt aliases
+    const openaiAuth = auth.openai || auth.codex || auth.chatgpt;
+    if (openaiAuth) {
+      // Handle both string format (just the token) and object format
+      if (typeof openaiAuth === 'string') {
+        return openaiAuth;
+      }
+      // Try access token first (OAuth), then regular token
+      if (openaiAuth.access) {
+        return openaiAuth.access;
+      }
+      if (openaiAuth.token) {
+        return openaiAuth.token;
+      }
+    }
+  } catch (error) {
+    console.warn('[TTSService] Failed to read auth file:', error.message);
+  }
+
+  return null;
+}
+
+class TTSService {
+  constructor() {
+    this._client = null;
+    this._lastApiKey = null;
+  }
+
+  _getClient() {
+    const apiKey = getOpenAIApiKey();
+
+    // If API key changed or client doesn't exist, create new client
+    if (apiKey && (!this._client || this._lastApiKey !== apiKey)) {
+      this._client = new OpenAI({ apiKey });
+      this._lastApiKey = apiKey;
+    }
+
+    return this._client;
+  }
+
+  isAvailable() {
+    return this._getClient() !== null;
+  }
+
+  /**
+   * Generate speech and return as a stream
+   */
+  async generateSpeechStream(options) {
+    const {
+      text,
+      voice = 'coral',
+      model = 'gpt-4o-mini-tts',
+      speed = 1.0,
+      instructions,
+      apiKey,
+      baseURL,
+    } = options;
+
+    const normalizedBaseURLResult = normalizeCustomOpenAIBaseURL(baseURL);
+    if (normalizedBaseURLResult.error) {
+      throw new Error(normalizedBaseURLResult.error);
+    }
+    const normalizedBaseURL = normalizedBaseURLResult.value;
+
+    // Use provided API key / baseURL or fall back to configured key
+    let client;
+    if (normalizedBaseURL || apiKey) {
+      const clientOpts = {};
+      if (apiKey) clientOpts.apiKey = apiKey;
+      if (!apiKey) clientOpts.apiKey = 'not-required';
+      if (normalizedBaseURL) clientOpts.baseURL = normalizedBaseURL;
+      client = new OpenAI(clientOpts);
+    } else {
+      client = this._getClient();
+    }
+
+    if (!client) {
+      throw new Error('TTS service not available. Configure OpenAI in OpenCode, provide an API key, or set a custom server URL in settings.');
+    }
+
+    if (!text.trim()) {
+      throw new Error('Text is required for TTS');
+    }
+
+    try {
+      // OpenAI-compatible servers (custom baseURL) may not support `instructions`
+      // or `response_format`, but do support `speed`. Send the safe subset.
+      const speechParams = normalizedBaseURL
+        ? { model, voice, input: text, speed }
+        : {
+            model,
+            voice,
+            input: text,
+            speed,
+            ...(instructions && { instructions }),
+            response_format: 'mp3',
+          };
+
+      console.log('[TTSService] Generating speech — model:', model, 'voice:', voice, 'baseURL:', normalizedBaseURL ?? '(openai)');
+      const response = await client.audio.speech.create(speechParams);
+
+      const arrayBuffer = await response.arrayBuffer();
+      return {
+        buffer: Buffer.from(arrayBuffer),
+        contentType: 'audio/mpeg',
+      };
+    } catch (error) {
+      console.error('[TTSService] Error generating speech:', error);
+      throw new Error(`Failed to generate speech: ${error.message || 'Unknown error'}`);
+    }
+  }
+
+  /**
+   * Generate speech and return as a buffer (for caching)
+   */
+  async generateSpeechBuffer(options) {
+    const client = this._getClient();
+    if (!client) {
+      throw new Error('OpenAI API key not configured. Set OPENAI_API_KEY environment variable or configure OpenAI in OpenCode.');
+    }
+
+    const {
+      text,
+      voice = 'coral',
+      model = 'gpt-4o-mini-tts',
+      speed = 1.0,
+      instructions
+    } = options;
+
+    try {
+      const response = await client.audio.speech.create({
+        model,
+        voice,
+        input: text,
+        speed,
+        ...(instructions && { instructions }),
+        response_format: 'mp3',
+      });
+
+      const arrayBuffer = await response.arrayBuffer();
+      return Buffer.from(arrayBuffer);
+    } catch (error) {
+      console.error('[TTSService] Error generating speech buffer:', error);
+      throw error;
+    }
+  }
+}
+
+// Export singleton instance
+export const ttsService = new TTSService();
+export { TTSService };

package/server/lib/tts/stt.js

@@ -0,0 +1,75 @@
+/**
+ * Server-side Speech-to-Text Service
+ *
+ * Proxies audio to any OpenAI-compatible transcription endpoint
+ * (e.g. faster-whisper, whisper.cpp) using the OpenAI Node SDK.
+ */
+
+import OpenAI, { toFile } from 'openai';
+import { normalizeCustomOpenAIBaseURL } from './base-url.js';
+
+/**
+ * Transcribe an audio buffer via an OpenAI-compatible /v1/audio/transcriptions endpoint.
+ *
+ * @param {object} opts
+ * @param {Buffer} opts.audioBuffer  - Raw audio bytes
+ * @param {string} opts.mimeType     - MIME type of the audio (e.g. 'audio/webm')
+ * @param {string} opts.model        - Model name accepted by the remote server
+ * @param {string} [opts.baseURL]    - Base URL of the compatible server (including /v1)
+ * @param {string} [opts.language]   - Optional BCP-47 language hint (e.g. 'en')
+ * @returns {Promise<string>} Transcribed text
+ */
+export async function transcribeAudio({ audioBuffer, mimeType, model, baseURL, language }) {
+  const normalizedBaseURLResult = normalizeCustomOpenAIBaseURL(baseURL);
+  if (normalizedBaseURLResult.error) {
+    throw new Error(normalizedBaseURLResult.error);
+  }
+
+  const normalizedBaseURL = normalizedBaseURLResult.value;
+  if (!normalizedBaseURL) {
+    throw new Error('Custom server URL is required');
+  }
+
+  const clientOpts = {
+    apiKey: process.env.OPENAI_API_KEY || 'not-required',
+  };
+  clientOpts.baseURL = normalizedBaseURL;
+
+  const client = new OpenAI(clientOpts);
+
+  // Derive a sensible filename extension from the MIME type so the server
+  // can infer the codec when it isn't explicit in the stream header.
+  const ext = mimeTypeToExt(mimeType);
+  const filename = `audio.${ext}`;
+
+  const file = await toFile(audioBuffer, filename, { type: mimeType });
+
+  const result = await client.audio.transcriptions.create({
+    file,
+    model,
+    response_format: 'json',
+    ...(language ? { language } : {}),
+  });
+
+  return result.text ?? '';
+}
+
+/**
+ * Map a MIME type to a file extension understood by Whisper servers.
+ * @param {string} mimeType
+ * @returns {string}
+ */
+function mimeTypeToExt(mimeType) {
+  const type = (mimeType || '').split(';')[0].trim().toLowerCase();
+  const map = {
+    'audio/webm': 'webm',
+    'audio/ogg': 'ogg',
+    'audio/wav': 'wav',
+    'audio/wave': 'wav',
+    'audio/mpeg': 'mp3',
+    'audio/mp4': 'mp4',
+    'audio/mp3': 'mp3',
+    'audio/flac': 'flac',
+  };
+  return map[type] ?? 'webm';
+}