clay-server 4.3.1-beta.5 → 4.4.0-beta.1

package/lib/speech-catalog.js

@@ -0,0 +1,25 @@
+// Provider IDs and capabilities are server-owned; clients cannot choose upstream URLs.
+var providers = [
+  { id: 'openai', name: 'OpenAI', icon: '/codex-avatar.png', url: 'https://platform.openai.com/api-keys' },
+  { id: 'elevenlabs', name: 'ElevenLabs', icon: '/icons/speech/elevenlabs.svg', url: 'https://elevenlabs.io/app/settings/api-keys' },
+  { id: 'soniox', name: 'Soniox', icon: '/icons/speech/soniox.png', url: 'https://console.soniox.com/' },
+  { id: 'groq', name: 'Groq', icon: '/icons/speech/groq.svg', url: 'https://console.groq.com/keys' },
+  { id: 'deepgram', name: 'Deepgram', icon: '/icons/speech/deepgram.svg', url: 'https://console.deepgram.com/' }
+];
+var models = [
+  ['openai-live', 'openai', 'GPT Live Transcribe', 'live', 'gpt-live-transcribe', 'Low-latency transcription with text appearing as you speak.'],
+  ['openai-transcribe', 'openai', 'GPT Transcribe', 'recorded', 'gpt-transcribe', 'Transcribes your recording after you stop.'],
+  ['openai-whisper', 'openai', 'Whisper', 'recorded', 'whisper-1', 'General-purpose multilingual transcription.'],
+  ['browser', 'browser', 'Browser speech', 'live', '', 'Uses your browser’s speech service. No API key required.'],
+  ['elevenlabs-live', 'elevenlabs', 'Scribe v2 Realtime', 'live', 'scribe_v2_realtime', 'Live multilingual dictation with partial text as you speak.'],
+  ['elevenlabs-file', 'elevenlabs', 'Scribe v2', 'recorded', 'scribe_v2', 'Multilingual transcription of completed recordings.'],
+  ['soniox-live', 'soniox', 'Soniox Realtime', 'live', 'stt-rt-v5', 'Live transcription with automatic language recognition.'],
+  ['soniox-file', 'soniox', 'Soniox Async', 'recorded', 'stt-async-v5', 'Asynchronous transcription of completed recordings.'],
+  ['groq-whisper', 'groq', 'Whisper Large v3', 'recorded', 'whisper-large-v3', 'Multilingual Whisper transcription hosted by Groq.'],
+  ['groq-turbo', 'groq', 'Whisper Turbo', 'recorded', 'whisper-large-v3-turbo', 'A faster Whisper variant hosted by Groq.'],
+  ['deepgram-live', 'deepgram', 'Nova-3', 'live', 'nova-3', 'Live transcription in your selected language.'],
+  ['deepgram-file', 'deepgram', 'Nova-3', 'recorded', 'nova-3', 'Recorded-audio transcription in your selected language.']
+].map(function (row) { return { id: row[0], provider: row[1], name: row[2], mode: row[3], model: row[4], description: row[5] }; });
+function findModel(id) { return models.find(function (model) { return model.id === id; }); }
+function findProvider(id) { return providers.find(function (provider) { return provider.id === id; }); }
+module.exports = { models: models, providers: providers, findModel: findModel, findProvider: findProvider };

package/lib/speech-openai-stream.js

@@ -0,0 +1,102 @@
+var providerError = require('./speech-transcribe').providerError;
+
+// Interpolate the shared 16 kHz PCM stream to OpenAI's required 24 kHz,
+// carrying sample position across chunks so chunk boundaries do not lose audio.
+function pcmResampler() {
+  var count = 0; var tick = 0; var previous = 0;
+  return function (bytes, flush) {
+    var samples = [];
+    for (var at = 0; at < bytes.length; at += 2) {
+      var current = bytes.readInt16LE(at);
+      while (tick <= count * 3) {
+        var fraction = (tick - (count - 1) * 3) / 3;
+        samples.push(count ? Math.round(previous + (current - previous) * fraction) : current);
+        tick += 2;
+      }
+      previous = current; count++;
+    }
+    if (flush) while (tick < count * 3) { samples.push(previous); tick += 2; }
+    var output = Buffer.alloc(samples.length * 2);
+    samples.forEach(function (sample, index) { output.writeInt16LE(sample, index * 2); });
+    return output;
+  };
+}
+
+function openOpenAIStream(model, key, language, events, Socket) {
+  var socket = new Socket('wss://api.openai.com/v1/realtime?intent=transcription', {
+    headers: { Authorization: 'Bearer ' + key }, handshakeTimeout: 15000, maxPayload: 1024 * 1024
+  });
+  var ended = false; var ready = false; var finishing = false; var timer;
+  var partial = ''; var itemId = null; var committedId = null; var completed = new Map();
+  var bytesSent = 0; var resample = pcmResampler();
+  function send(message) { socket.send(JSON.stringify(message)); }
+  function fail(error) {
+    if (ended) return;
+    ended = true; clearTimeout(timer); socket.terminate(); events.error(error);
+  }
+  function done(text) {
+    if (ended) return;
+    ended = true; clearTimeout(timer); events.done(text.trim()); socket.close();
+  }
+  function failure(message) {
+    var code = message.error && message.error.code;
+    if (code === 'invalid_api_key') return providerError(401);
+    if (code === 'insufficient_quota' || code === 'rate_limit_exceeded') return providerError(429);
+    if (code === 'model_not_found') return new Error('GPT Live Transcribe is unavailable for this OpenAI project. Check model access.');
+    return new Error('OpenAI could not complete live transcription. Check your API key, model access and connection.');
+  }
+  function append(bytes) {
+    if (bytes.length) send({ type: 'input_audio_buffer.append', audio: bytes.toString('base64') });
+  }
+  socket.on('open', function () {
+    if (ended) return;
+    send({ type: 'session.update', session: { type: 'transcription', audio: { input: {
+      format: { type: 'audio/pcm', rate: 24000 },
+      transcription: { model: model.model, languages: [language.toLowerCase().split('-')[0]] },
+      turn_detection: null
+    } } } });
+  });
+  socket.on('message', function (raw) {
+    if (ended) return;
+    try {
+      var message = JSON.parse(raw.toString());
+      if (message.type === 'error' || message.type === 'conversation.item.input_audio_transcription.failed') { fail(failure(message)); return; }
+      if (message.type === 'session.updated' && !ready) { ready = true; events.ready(); }
+      else if (message.type === 'conversation.item.input_audio_transcription.delta') {
+        if (!itemId) itemId = message.item_id;
+        if (message.item_id !== itemId) return;
+        partial += message.delta || ''; events.text(partial);
+      } else if (message.type === 'input_audio_buffer.committed') {
+        committedId = message.item_id;
+        if (finishing && completed.has(committedId)) done(completed.get(committedId));
+      } else if (message.type === 'conversation.item.input_audio_transcription.completed') {
+        if (!itemId) itemId = message.item_id;
+        if (message.item_id !== itemId) return;
+        completed.set(message.item_id, message.transcript || '');
+        if (finishing && message.item_id === committedId) done(message.transcript || '');
+      }
+    } catch (error) { fail(providerError(502)); }
+  });
+  socket.on('error', function () { fail(new Error('Could not connect to OpenAI live transcription. Please try again.')); });
+  socket.on('unexpected-response', function (request, response) { response.resume(); fail(providerError(response.statusCode)); });
+  socket.on('close', function () { if (!ended) fail(new Error('The OpenAI live connection ended unexpectedly. Your draft has been preserved.')); });
+  return {
+    audio: function (bytes) {
+      if (ended || finishing || !ready || socket.readyState !== 1) return;
+      if (socket.bufferedAmount > 1024 * 1024) { fail(new Error('The connection is too slow for live transcription.')); return; }
+      bytesSent += bytes.length; append(resample(bytes, false));
+    },
+    finish: function () {
+      if (ended || finishing || !ready) return;
+      finishing = true;
+      if (!bytesSent) { done(''); return; }
+      // OpenAI requires at least 100 ms per commit; pad only very short captures.
+      if (bytesSent < 3200) append(resample(Buffer.alloc(3200 - bytesSent), false));
+      append(resample(Buffer.alloc(0), true));
+      timer = setTimeout(function () { fail(new Error('OpenAI transcription timed out. Your draft has been preserved.')); }, 20000);
+      send({ type: 'input_audio_buffer.commit' });
+    },
+    abort: function () { ended = true; clearTimeout(timer); socket.terminate(); }
+  };
+}
+module.exports = { openOpenAIStream: openOpenAIStream, pcmResampler: pcmResampler };

package/lib/speech-store.js

@@ -0,0 +1,71 @@
+var fs = require('fs');
+var path = require('path');
+var crypto = require('crypto');
+var catalog = require('./speech-catalog');
+
+function attachSpeechStore(root) {
+  var directory = path.join(root, 'speech');
+  function prepare() { fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); }
+  function filename(owner) {
+    if (typeof owner !== 'string' || !/^[a-zA-Z0-9_-]{1,128}$/.test(owner)) throw new Error('Invalid owner');
+    return path.join(directory, owner + '.json');
+  }
+  function read(owner) {
+    try { return JSON.parse(fs.readFileSync(filename(owner), 'utf8')); }
+    catch (error) { if (error.code === 'ENOENT') return { selected: 'browser', keys: {} }; throw error; }
+  }
+  function masterKey() {
+    prepare();
+    var file = path.join(directory, '.key');
+    try { fs.writeFileSync(file, crypto.randomBytes(32), { flag: 'wx', mode: 0o600 }); }
+    catch (error) { if (error.code !== 'EEXIST') throw error; }
+    return fs.readFileSync(file);
+  }
+  function encrypt(owner, provider, text) {
+    var iv = crypto.randomBytes(12);
+    var cipher = crypto.createCipheriv('aes-256-gcm', masterKey(), iv);
+    cipher.setAAD(Buffer.from(owner + ':' + provider));
+    var data = Buffer.concat([cipher.update(text, 'utf8'), cipher.final()]);
+    return [iv, cipher.getAuthTag(), data].map(function (part) { return part.toString('base64'); }).join('.');
+  }
+  function key(owner, provider) {
+    var value = (read(owner).keys || {})[provider];
+    if (!value) return null;
+    var parts = value.split('.').map(function (part) { return Buffer.from(part, 'base64'); });
+    var cipher = crypto.createDecipheriv('aes-256-gcm', masterKey(), parts[0]);
+    cipher.setAAD(Buffer.from(owner + ':' + provider)); cipher.setAuthTag(parts[1]);
+    return Buffer.concat([cipher.update(parts[2]), cipher.final()]).toString('utf8');
+  }
+  function view(owner) {
+    var data = read(owner);
+    return { selected: data.selected || 'browser', language: data.language || 'en-US', configured: Object.keys(data.keys || {}), models: catalog.models, providers: catalog.providers };
+  }
+  function update(owner, change) {
+    var data = read(owner); data.keys = data.keys || {};
+    if (change.provider !== undefined) {
+      if (!catalog.findProvider(change.provider)) throw new Error('Unknown speech provider');
+      if (change.key === null) delete data.keys[change.provider];
+      else {
+        if (typeof change.key !== 'string' || !change.key.trim() || change.key.length > 4096 || /[\r\n]/.test(change.key)) throw new Error('Enter a valid API key');
+        data.keys[change.provider] = encrypt(owner, change.provider, change.key.trim());
+      }
+    }
+    if (change.language !== undefined) {
+      if (typeof change.language !== 'string' || !/^[a-z]{2,3}(?:-[a-zA-Z]{2,8})?$/.test(change.language)) throw new Error('Invalid speech language');
+      data.language = change.language;
+    }
+    if (change.selected !== undefined) {
+      var model = catalog.findModel(change.selected);
+      if (!model || (model.provider !== 'browser' && !data.keys[model.provider])) throw new Error('Connect the provider before selecting this model');
+      data.selected = model.id;
+    }
+    // Keep selection after key removal so there is no silent provider fallback.
+    prepare();
+    var file = filename(owner); var temporary = file + '.' + crypto.randomBytes(8).toString('hex') + '.tmp';
+    try { fs.writeFileSync(temporary, JSON.stringify(data), { mode: 0o600 }); fs.renameSync(temporary, file); }
+    finally { try { fs.unlinkSync(temporary); } catch (error) { if (error.code !== 'ENOENT') throw error; } }
+    return view(owner);
+  }
+  return { view: view, update: update, key: key };
+}
+module.exports = { attachSpeechStore: attachSpeechStore };

package/lib/speech-stream.js

@@ -0,0 +1,108 @@
+var WebSocket = require('ws');
+var providerError = require('./speech-transcribe').providerError;
+
+function streamError(model, message) {
+  var status = Number(message.error_code) || 502;
+  if (model.provider !== 'soniox') return providerError(status);
+  var reasons = {
+    unauthenticated: 'Soniox rejected the API key. Check the key in Voice input settings.',
+    permission_denied: 'The Soniox key does not have permission for live transcription. Check its permissions in Soniox Console.',
+    organization_balance_exhausted: 'Your Soniox credit balance is empty. Add credits in Soniox Console to use voice input.',
+    organization_monthly_budget_exhausted: 'Your Soniox organization has reached its monthly budget. Check its billing limits.',
+    project_monthly_budget_exhausted: 'Your Soniox project has reached its monthly budget. Check its billing limits.',
+    model_not_available: 'The selected Soniox model is unavailable for your project or region. Check model access in Soniox Console.',
+    invalid_request: 'Soniox rejected the live transcription configuration. Try another voice model.',
+    limit_exceeded: 'Your Soniox request limit was reached. Wait for other transcriptions to finish and try again.',
+    request_timeout: 'Soniox timed out waiting for microphone audio. Check your microphone and connection, then try again.',
+    service_unavailable: 'Soniox is temporarily unavailable. Please try again later.'
+  };
+  var reason = Object.prototype.hasOwnProperty.call(reasons, message.error_type) ? reasons[message.error_type] : null;
+  if (!reason && status === 402) reason = 'Soniox requires available credits and budget. Check billing in Soniox Console.';
+  return reason ? new Error(reason) : providerError(status);
+}
+
+function openSpeechStream(model, key, language, events, Socket) {
+  Socket = Socket || WebSocket;
+  if (model.provider === 'openai') return require('./speech-openai-stream').openOpenAIStream(model, key, language, events, Socket);
+  var code = language.split('-')[0];
+  var url; var options = { handshakeTimeout: 15000, maxPayload: 1024 * 1024 };
+  if (model.provider === 'soniox') url = 'wss://stt-rt.soniox.com/transcribe-websocket';
+  else if (model.provider === 'elevenlabs') {
+    url = 'wss://api.elevenlabs.io/v1/speech-to-text/realtime?model_id=scribe_v2_realtime&audio_format=pcm_16000&commit_strategy=manual&language_code=' + encodeURIComponent(code);
+    options.headers = { 'xi-api-key': key };
+  } else if (model.provider === 'deepgram') {
+    url = 'wss://api.deepgram.com/v1/listen?model=nova-3&encoding=linear16&sample_rate=16000&channels=1&interim_results=true&smart_format=true&language=' + encodeURIComponent(code);
+    options.headers = { Authorization: 'Token ' + key };
+  } else throw new Error('This model does not support live transcription');
+  var socket = new Socket(url, options);
+  var finalText = ''; var ended = false; var finishing = false; var finishTimer;
+  function diagnostic(stage, details) { if (events.diagnostic) events.diagnostic(stage, details); }
+  function fail(error) { if (!ended) { ended = true; clearTimeout(finishTimer); socket.terminate(); events.error(error); } }
+  function done() { if (!ended) { ended = true; clearTimeout(finishTimer); events.done(finalText.trim()); socket.close(); } }
+  function emit(partial) { events.text((finalText + partial).trim()); }
+  socket.on('open', function () {
+    if (ended) return;
+    diagnostic('upstream-websocket-open');
+    if (model.provider === 'soniox') socket.send(JSON.stringify({ api_key: key, model: model.model,
+      audio_format: 'pcm_s16le', sample_rate: 16000, num_channels: 1, language_hints: [code], enable_endpoint_detection: true }));
+    if (model.provider === 'soniox') diagnostic('soniox-config-sent');
+    if (model.provider !== 'elevenlabs') events.ready();
+  });
+  socket.on('message', function (raw) {
+    if (ended) return;
+    try {
+      var message = JSON.parse(raw.toString());
+      if (message.error || message.error_code || message.error_type || message.type === 'Error' || /error|rate_limited|quota_exceeded/.test(message.message_type || '')) {
+        diagnostic('upstream-error-response', { status: Number(message.error_code) || 502 });
+        fail(streamError(model, message)); return;
+      }
+      if (model.provider === 'soniox') {
+        var partial = '';
+        (message.tokens || []).forEach(function (token) {
+          if (/^<.*>$/.test(token.text)) return;
+          if (token.is_final) finalText += token.text; else partial += token.text;
+        });
+        emit(partial);
+        if (message.finished) done();
+      } else if (model.provider === 'elevenlabs') {
+        if (message.message_type === 'session_started') events.ready();
+        if (message.message_type === 'partial_transcript') emit(message.text || '');
+        if (message.message_type === 'committed_transcript') {
+          finalText += (message.text || '') + ' '; emit('');
+          if (finishing) done();
+        }
+      } else if (message.type === 'Results') {
+        var text = message.channel.alternatives[0].transcript || '';
+        if (message.is_final) { finalText += text + (text ? ' ' : ''); emit(''); }
+        else emit(text);
+      } else if (message.type === 'Metadata' && finishing) done();
+    } catch (error) { fail(providerError(502)); }
+  });
+  socket.on('error', function () { diagnostic('upstream-transport-error'); fail(providerError(502)); });
+  socket.on('unexpected-response', function (request, response) { diagnostic('upstream-handshake-rejected', { status: response.statusCode }); response.resume(); fail(providerError(response.statusCode)); });
+  socket.on('close', function (code) {
+    diagnostic('upstream-websocket-close', { status: code });
+    if (!ended) {
+      if (finishing && model.provider === 'deepgram') done();
+      else fail(new Error('The live connection ended unexpectedly. Your draft has been preserved.'));
+    }
+  });
+  return {
+    audio: function (bytes) {
+      if (ended || finishing || socket.readyState !== 1) return;
+      if (socket.bufferedAmount > 1024 * 1024) { fail(new Error('The connection is too slow for live transcription.')); return; }
+      if (model.provider === 'elevenlabs') socket.send(JSON.stringify({ message_type: 'input_audio_chunk', audio_base_64: bytes.toString('base64'), sample_rate: 16000 }));
+      else socket.send(bytes);
+    },
+    finish: function () {
+      if (ended || finishing) return;
+      finishing = true;
+      finishTimer = setTimeout(function () { fail(new Error('Transcription timed out. Your draft has been preserved.')); }, 20000);
+      if (model.provider === 'soniox') socket.send('');
+      else if (model.provider === 'deepgram') socket.send(JSON.stringify({ type: 'CloseStream' }));
+      else socket.send(JSON.stringify({ message_type: 'input_audio_chunk', audio_base_64: '', commit: true, sample_rate: 16000 }));
+    },
+    abort: function () { ended = true; clearTimeout(finishTimer); socket.terminate(); }
+  };
+}
+module.exports = { openSpeechStream: openSpeechStream };

package/lib/speech-transcribe.js

@@ -0,0 +1,63 @@
+// Recorded audio adapters. No provider response body (which may contain private data) is logged.
+function providerError(status) {
+  var error = new Error(status === 401 || status === 403 ? 'The provider rejected your API key. Check its permissions in Voice input settings.' :
+    status === 429 ? 'The provider limit was reached. Check your quota or try again later.' : 'The transcription provider could not complete the request. Please try again.');
+  error.status = 502;
+  return error;
+}
+async function transcribe(model, key, audio, language, signal, fetcher) {
+  fetcher = fetcher || fetch;
+  var languageCode = (language || 'en-US').split('-')[0];
+  async function request(url, options) {
+    var response = await fetcher(url, Object.assign({}, options, { signal: signal }));
+    if (!response.ok) throw providerError(response.status);
+    return response;
+  }
+  function form() { var data = new FormData(); data.append('file', new Blob([audio], { type: 'audio/wav' }), 'voice.wav'); return data; }
+  if (model.provider === 'openai' || model.provider === 'groq') {
+    var data = form(); data.append('model', model.model); data.append(model.model === 'gpt-transcribe' ? 'languages[]' : 'language', languageCode);
+    var endpoint = model.provider === 'openai' ? 'https://api.openai.com/v1/audio/transcriptions' : 'https://api.groq.com/openai/v1/audio/transcriptions';
+    var response = await request(endpoint, { method: 'POST', headers: { Authorization: 'Bearer ' + key }, body: data });
+    return (await response.json()).text;
+  }
+  if (model.provider === 'elevenlabs') {
+    var data = form(); data.append('model_id', model.model); data.append('language_code', languageCode);
+    data.append('tag_audio_events', 'false'); data.append('diarize', 'false');
+    var response = await request('https://api.elevenlabs.io/v1/speech-to-text', { method: 'POST', headers: { 'xi-api-key': key }, body: data });
+    return (await response.json()).text;
+  }
+  if (model.provider === 'deepgram') {
+    var url = 'https://api.deepgram.com/v1/listen?model=nova-3&smart_format=true&language=' + encodeURIComponent(languageCode);
+    var response = await request(url, { method: 'POST', headers: { Authorization: 'Token ' + key, 'Content-Type': 'audio/wav' }, body: audio });
+    return (await response.json()).results.channels[0].alternatives[0].transcript;
+  }
+  if (model.provider !== 'soniox') throw new Error('Unsupported speech provider');
+  var base = 'https://api.soniox.com/v1';
+  var headers = { Authorization: 'Bearer ' + key };
+  var fileId; var transcriptionId;
+  try {
+    var uploaded = await request(base + '/files', { method: 'POST', headers: headers, body: form() });
+    fileId = (await uploaded.json()).id;
+    var created = await request(base + '/transcriptions', { method: 'POST', headers: Object.assign({}, headers, { 'Content-Type': 'application/json' }),
+      body: JSON.stringify({ model: model.model, file_id: fileId, language_hints: [languageCode] }) });
+    transcriptionId = (await created.json()).id;
+    for (;;) {
+      var result = await request(base + '/transcriptions/' + encodeURIComponent(transcriptionId), { headers: headers });
+      var state = await result.json();
+      if (state.status === 'error') throw providerError(502);
+      if (state.status === 'completed') break;
+      await require('timers/promises').setTimeout(750, undefined, { signal: signal });
+    }
+    var transcript = await request(base + '/transcriptions/' + encodeURIComponent(transcriptionId) + '/transcript', { headers: headers });
+    return (await transcript.json()).text;
+  } finally {
+    // Cleanup uses its own bounded signal, including when the user cancels the transcription.
+    var paths = [];
+    if (transcriptionId) paths.push('/transcriptions/' + encodeURIComponent(transcriptionId));
+    if (fileId) paths.push('/files/' + encodeURIComponent(fileId));
+    await Promise.allSettled(paths.map(function (path) {
+      return fetcher(base + path, { method: 'DELETE', headers: headers, signal: AbortSignal.timeout(5000) });
+    }));
+  }
+}
+module.exports = { transcribe: transcribe, providerError: providerError };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clay-server",
-  "version": "4.3.1-beta.5",
+  "version": "4.4.0-beta.1",
   "description": "Self-hosted team workspace for Claude Code and Codex. Multi-user, browser-based, with persistent AI mates.",
   "bin": {
     "clay-server": "./bin/cli.js"