studio-engine 0.119.1 → 0.121.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +61 -0
- data/README.md +3 -0
- data/app/controllers/studio/profiles_controller.rb +56 -0
- data/app/controllers/studio/voice_takes_controller.rb +93 -0
- data/app/javascript/studio/controllers/microphone_chip_controller.js +21 -0
- data/app/javascript/studio/controllers/voice_profile_controller.js +20 -0
- data/app/javascript/studio/controllers/voice_recorder_controller.js +29 -0
- data/app/javascript/studio/level_meter.js +167 -0
- data/app/javascript/studio/microphone.js +250 -0
- data/app/javascript/studio/microphone_chip.js +370 -0
- data/app/javascript/studio/stimulus.js +10 -2
- data/app/javascript/studio/voice_profile.js +444 -0
- data/app/javascript/studio/voice_recorder.js +352 -0
- data/app/views/studio/profiles/_voice_section.html.erb +108 -0
- data/app/views/studio/voice/_microphone_chip.html.erb +45 -0
- data/app/views/studio/voice/_microphone_help.html.erb +63 -0
- data/app/views/studio/voice/_recorder.html.erb +34 -0
- data/app/views/studio/voice/_styles.html.erb +90 -0
- data/db/migrate/20261029120000_add_voice_profile_to_users.rb +44 -0
- data/lib/studio/engine.rb +10 -0
- data/lib/studio/profile_sections.rb +11 -0
- data/lib/studio/version.rb +1 -1
- data/lib/studio/voice_capture.rb +95 -0
- data/lib/studio/voice_match.rb +238 -0
- data/lib/studio/voice_transcription.rb +404 -0
- data/lib/studio.rb +57 -0
- metadata +20 -2
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "net/http"
|
|
5
|
+
require "openssl"
|
|
6
|
+
require "securerandom"
|
|
7
|
+
require "stringio"
|
|
8
|
+
require "uri"
|
|
9
|
+
|
|
10
|
+
# Understand: one finished take (the recorder's blob and its mime type, plus the
|
|
11
|
+
# speaker embedding the browser made of it) becomes {text, speaker, score}.
|
|
12
|
+
#
|
|
13
|
+
# The server half of the voice layer (docs/VOICE_TRANSCRIPTION.md). The capture
|
|
14
|
+
# half (docs/VOICE_CAPTURE.md) hands the page a take; the page posts it here,
|
|
15
|
+
# through Studio::VoiceTakesController (POST /voice/takes) or by calling
|
|
16
|
+
# Studio::VoiceTranscription.hear itself.
|
|
17
|
+
#
|
|
18
|
+
# heard = Studio::VoiceTranscription.hear(audio: io, mime_type: "audio/webm",
|
|
19
|
+
# user: current_user,
|
|
20
|
+
# embedding: vector, model: model_id)
|
|
21
|
+
# heard.to_h # => {"text"=>"...", "speaker"=>"Pat Studio", "score"=>0.88,
|
|
22
|
+
# # "identified"=>true, "check"=>"matched"}
|
|
23
|
+
#
|
|
24
|
+
# MOCK-FIRST, as Voicebox runs it (config/initializers/voicebox.rb, Adapters):
|
|
25
|
+
# the transcriber is chosen by name, production defaults to the real one and
|
|
26
|
+
# everywhere else to the mock, and a real adapter raises LiveCallAttempted
|
|
27
|
+
# before it opens a socket while VOICE_NO_LIVE_CALLS is armed (always, in a test
|
|
28
|
+
# environment). Ported from Voicebox's Transcription::OpenAI and
|
|
29
|
+
# Transcription::Mock.
|
|
30
|
+
#
|
|
31
|
+
# WHO SPOKE is Studio::VoiceMatch against the SIGNED-IN USER'S OWN saved voice
|
|
32
|
+
# (users.voice_embedding), compared only when both vectors came from the same
|
|
33
|
+
# embedder revision. Named people without an account are a later card.
|
|
34
|
+
#
|
|
35
|
+
# NOTHING IS KEPT. The audio is read once, sent to the transcriber and dropped;
|
|
36
|
+
# no vector or audio is logged (the engine's parameter filter masks
|
|
37
|
+
# voice[embedding] and voice[audio]). Pure Ruby apart from the optional Rails
|
|
38
|
+
# logger and environment, so the unit suite runs it without an app.
|
|
39
|
+
module Studio
|
|
40
|
+
module VoiceTranscription
|
|
41
|
+
# A transcriber call that failed: the message carries the vendor's words.
|
|
42
|
+
class Error < StandardError; end
|
|
43
|
+
|
|
44
|
+
# The vendor answered a non-2xx; the status rides along so a caller can tell
|
|
45
|
+
# an auth failure from a model that fell over.
|
|
46
|
+
class HttpError < Error
|
|
47
|
+
attr_reader :status
|
|
48
|
+
|
|
49
|
+
def initialize(message, status:)
|
|
50
|
+
super(message)
|
|
51
|
+
@status = status
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# A real transcriber chosen without its credential, or a name that is no
|
|
56
|
+
# adapter. The page shows NOT_CONFIGURED_MESSAGE; the cause is in the log.
|
|
57
|
+
class NotConfigured < Error; end
|
|
58
|
+
|
|
59
|
+
# DELIBERATELY OUTSIDE StandardError (Voicebox's Adapters::LiveCallAttempted):
|
|
60
|
+
# every caller turns a StandardError into a friendly failure, so a trap that
|
|
61
|
+
# was one would be swallowed by the code it guards and a test that reached
|
|
62
|
+
# a real vendor would pass quietly.
|
|
63
|
+
class LiveCallAttempted < Exception; end # rubocop:disable Lint/InheritException
|
|
64
|
+
|
|
65
|
+
NO_LIVE_CALLS_ENV = "VOICE_NO_LIVE_CALLS"
|
|
66
|
+
TRANSCRIBER_ENV = "STUDIO_VOICE_TRANSCRIBER"
|
|
67
|
+
API_KEY_ENV = "OPENAI_API_KEY"
|
|
68
|
+
|
|
69
|
+
# What a page shows when the app has no transcriber key. Says what is wrong
|
|
70
|
+
# without naming the variable to a visitor.
|
|
71
|
+
NOT_CONFIGURED_MESSAGE = "Voice isn't set up on this app yet, so I can't hear recordings. Type instead for now."
|
|
72
|
+
|
|
73
|
+
# The bounds on one take. The recorder stops itself at 60 seconds by default
|
|
74
|
+
# (studio/voice_recorder max_ms); a minute of opus is about half a megabyte,
|
|
75
|
+
# so 10 MB is far above any honest take and well under the vendor's 25 MB.
|
|
76
|
+
MAX_BYTES = 10 * 1024 * 1024
|
|
77
|
+
MAX_DURATION_MS = 120_000
|
|
78
|
+
|
|
79
|
+
# The audio types a take may be, as MediaRecorder names them (without the
|
|
80
|
+
# ";codecs=" parameter), and the extension the vendor reads the format from.
|
|
81
|
+
EXTENSIONS = {
|
|
82
|
+
"audio/webm" => "webm",
|
|
83
|
+
"video/webm" => "webm",
|
|
84
|
+
"audio/ogg" => "ogg",
|
|
85
|
+
"audio/mp4" => "mp4",
|
|
86
|
+
"audio/m4a" => "m4a",
|
|
87
|
+
"audio/x-m4a" => "m4a",
|
|
88
|
+
"audio/mpeg" => "mp3",
|
|
89
|
+
"audio/wav" => "wav",
|
|
90
|
+
"audio/x-wav" => "wav"
|
|
91
|
+
}.freeze
|
|
92
|
+
|
|
93
|
+
# The label a take gets when it is not the signed-in user's saved voice. The
|
|
94
|
+
# agent is told, and the message still sends (epic default).
|
|
95
|
+
SOMEONE_NEW = "Someone new"
|
|
96
|
+
|
|
97
|
+
# Why the speaker is (or is not) named, in Heard#check:
|
|
98
|
+
# matched the user's own voice, at or above VoiceMatch::THRESHOLD
|
|
99
|
+
# below_threshold compared, and the closest score fell short
|
|
100
|
+
# no_profile the user has no saved voice to compare with
|
|
101
|
+
# embedder_mismatch the take and the saved voice came from different
|
|
102
|
+
# embedders or revisions, so they were not compared
|
|
103
|
+
# no_embedding the take carried no usable speaker embedding
|
|
104
|
+
CHECKS = %w[matched below_threshold no_profile embedder_mismatch no_embedding].freeze
|
|
105
|
+
|
|
106
|
+
# Languages written in the Latin alphabet (Voicebox's RecordingPipeline).
|
|
107
|
+
LATIN_LANGUAGES = %w[en es fr de it pt nl].freeze
|
|
108
|
+
# What speech models write for silence, whole, lowercased, unpunctuated.
|
|
109
|
+
SILENCE_PHRASES = [
|
|
110
|
+
"thank you", "thanks", "thank you for watching", "thanks for watching",
|
|
111
|
+
"thank you for listening", "thanks for listening", "bye", "bye bye", "you"
|
|
112
|
+
].freeze
|
|
113
|
+
FILLERS = %w[um umm uh uhh hmm mm mhm ah er erm].freeze
|
|
114
|
+
|
|
115
|
+
Heard = Struct.new(:text, :speaker, :score, :identified, :check, keyword_init: true) do
|
|
116
|
+
def identified? = identified
|
|
117
|
+
|
|
118
|
+
def to_h
|
|
119
|
+
{ "text" => text, "speaker" => speaker, "score" => score, "identified" => identified, "check" => check }
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# The real speech-to-text: OpenAI's POST /v1/audio/transcriptions, one
|
|
124
|
+
# multipart file. Ported from Voicebox's Transcription::OpenAI: the model is
|
|
125
|
+
# told the language (left to guess, a model handed near-silence answers a
|
|
126
|
+
# stock word in any language), and a failure other than auth is retried once
|
|
127
|
+
# on the fallback model from the start of the audio.
|
|
128
|
+
class OpenAI
|
|
129
|
+
ENDPOINT = "https://api.openai.com/v1/audio/transcriptions"
|
|
130
|
+
OPEN_TIMEOUT = 10
|
|
131
|
+
READ_TIMEOUT = 120
|
|
132
|
+
AUTH_FAILURES = [401, 403].freeze
|
|
133
|
+
|
|
134
|
+
attr_reader :model, :fallback_model, :language, :endpoint
|
|
135
|
+
|
|
136
|
+
def initialize(api_key: VoiceTranscription.api_key, model: Studio.voice_transcribe_model,
|
|
137
|
+
fallback_model: Studio.voice_transcribe_fallback_model,
|
|
138
|
+
language: Studio.voice_transcribe_language, endpoint: ENDPOINT)
|
|
139
|
+
@api_key = api_key
|
|
140
|
+
@model = model.to_s
|
|
141
|
+
@fallback_model = fallback_model.to_s.strip.empty? ? nil : fallback_model.to_s
|
|
142
|
+
@language = language.to_s.strip.empty? ? nil : language.to_s
|
|
143
|
+
@endpoint = endpoint
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def configured? = !@api_key.to_s.strip.empty?
|
|
147
|
+
|
|
148
|
+
# io: the audio, at its start. Returns the transcript, stripped.
|
|
149
|
+
def transcribe(io:, filename:, content_type:)
|
|
150
|
+
raise NotConfigured, "#{API_KEY_ENV} is not set, so this app cannot transcribe voice" unless configured?
|
|
151
|
+
|
|
152
|
+
request_text(model, io, filename, content_type)
|
|
153
|
+
rescue HttpError => e
|
|
154
|
+
raise if fallback_model.nil? || fallback_model == model || AUTH_FAILURES.include?(e.status)
|
|
155
|
+
|
|
156
|
+
VoiceTranscription.log_warn("#{model} transcription failed (#{e.message}); retrying on #{fallback_model}")
|
|
157
|
+
io.rewind
|
|
158
|
+
request_text(fallback_model, io, filename, content_type)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# The multipart fields, apart from the socket so a test reads them.
|
|
162
|
+
def form_fields(model_name, io, filename, content_type)
|
|
163
|
+
fields = [
|
|
164
|
+
["file", io, { filename: filename, content_type: content_type }],
|
|
165
|
+
["model", model_name],
|
|
166
|
+
["response_format", "json"]
|
|
167
|
+
]
|
|
168
|
+
fields << ["language", language] if language
|
|
169
|
+
fields
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# The request, built but not sent: a multipart body with a Content-Length
|
|
173
|
+
# (not chunked), the bearer key, JSON accepted.
|
|
174
|
+
def build_request(model_name, io, filename, content_type)
|
|
175
|
+
uri = URI.parse(endpoint)
|
|
176
|
+
boundary = "studio-voice-#{SecureRandom.hex(12)}"
|
|
177
|
+
request = Net::HTTP::Post.new(uri)
|
|
178
|
+
request["Authorization"] = "Bearer #{@api_key}"
|
|
179
|
+
request["Accept"] = "application/json"
|
|
180
|
+
request["Content-Type"] = "multipart/form-data; boundary=#{boundary}"
|
|
181
|
+
request.body = VoiceTranscription.multipart(form_fields(model_name, io, filename, content_type), boundary)
|
|
182
|
+
request
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
private
|
|
186
|
+
|
|
187
|
+
def request_text(model_name, io, filename, content_type)
|
|
188
|
+
perform(model_name, io, filename, content_type).fetch("text", "").to_s.strip
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# The only place a socket opens, so the only place the trap sits.
|
|
192
|
+
def perform(model_name, io, filename, content_type)
|
|
193
|
+
uri = URI.parse(endpoint)
|
|
194
|
+
VoiceTranscription.guard_live_call!("OpenAI", host: uri.host)
|
|
195
|
+
|
|
196
|
+
request = build_request(model_name, io, filename, content_type)
|
|
197
|
+
response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
|
|
198
|
+
open_timeout: OPEN_TIMEOUT, read_timeout: READ_TIMEOUT) do |http|
|
|
199
|
+
http.request(request)
|
|
200
|
+
end
|
|
201
|
+
interpret(response, model_name)
|
|
202
|
+
rescue Timeout::Error, SystemCallError, IOError, OpenSSL::SSL::SSLError, SocketError => e
|
|
203
|
+
raise HttpError.new("OpenAI #{model_name} could not be reached (#{e.class})", status: 0)
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def interpret(response, model_name)
|
|
207
|
+
parsed = begin
|
|
208
|
+
JSON.parse(response.body.to_s)
|
|
209
|
+
rescue JSON::ParserError
|
|
210
|
+
{}
|
|
211
|
+
end
|
|
212
|
+
parsed = {} unless parsed.is_a?(Hash)
|
|
213
|
+
return parsed if response.is_a?(Net::HTTPSuccess)
|
|
214
|
+
|
|
215
|
+
detail = parsed.dig("error", "message") || response.body.to_s[0, 200]
|
|
216
|
+
raise HttpError.new("OpenAI #{model_name} answered HTTP #{response.code}: #{detail}", status: response.code.to_i)
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# The default everywhere but production: no network, a fixed sentence. A
|
|
221
|
+
# test swaps the answer with Mock.reply = ->(**) { "..." } and resets with
|
|
222
|
+
# Mock.reset!. It records what it was handed (sizes, never the bytes).
|
|
223
|
+
class Mock
|
|
224
|
+
DEFAULT_TEXT = "This is a mock transcript of your voice."
|
|
225
|
+
|
|
226
|
+
class << self
|
|
227
|
+
attr_accessor :reply
|
|
228
|
+
attr_writer :calls
|
|
229
|
+
|
|
230
|
+
def calls = (@calls ||= [])
|
|
231
|
+
|
|
232
|
+
def reset!
|
|
233
|
+
self.reply = nil
|
|
234
|
+
self.calls = []
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def configured? = true
|
|
239
|
+
|
|
240
|
+
def transcribe(io:, filename:, content_type:)
|
|
241
|
+
bytes = io.read.to_s.bytesize
|
|
242
|
+
self.class.calls << { filename: filename, content_type: content_type, bytes: bytes }
|
|
243
|
+
reply = self.class.reply
|
|
244
|
+
reply.nil? ? DEFAULT_TEXT : reply.call(filename: filename, content_type: content_type, bytes: bytes).to_s
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
ADAPTERS = { "openai" => OpenAI, "mock" => Mock }.freeze
|
|
249
|
+
|
|
250
|
+
module_function
|
|
251
|
+
|
|
252
|
+
# The transcriber's name: Studio.voice_transcriber when the host set one,
|
|
253
|
+
# else STUDIO_VOICE_TRANSCRIBER, else "openai" in production and "mock"
|
|
254
|
+
# everywhere else.
|
|
255
|
+
def adapter_name
|
|
256
|
+
configured = Studio.voice_transcriber.to_s.strip
|
|
257
|
+
return configured unless configured.empty?
|
|
258
|
+
|
|
259
|
+
from_env = ENV[TRANSCRIBER_ENV].to_s.strip
|
|
260
|
+
return from_env unless from_env.empty?
|
|
261
|
+
|
|
262
|
+
rails_env == "production" ? "openai" : "mock"
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
def transcriber(name = adapter_name)
|
|
266
|
+
klass = ADAPTERS.fetch(name.to_s) do
|
|
267
|
+
raise NotConfigured, "Studio.voice_transcriber=#{name.inspect} names no adapter; one of: #{ADAPTERS.keys.join(', ')}"
|
|
268
|
+
end
|
|
269
|
+
klass.new
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
# Studio.voice_openai_api_key, else OPENAI_API_KEY (Voicebox's name).
|
|
273
|
+
def api_key
|
|
274
|
+
key = Studio.voice_openai_api_key.to_s.strip
|
|
275
|
+
key.empty? ? ENV[API_KEY_ENV].to_s.strip : key
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# Armed when VOICE_NO_LIVE_CALLS=1, and in any Rails test environment unless
|
|
279
|
+
# it is set to 0. A call to this machine (a stub server) is not live.
|
|
280
|
+
def live_calls_blocked?
|
|
281
|
+
flag = ENV[NO_LIVE_CALLS_ENV].to_s.strip
|
|
282
|
+
return true if flag == "1"
|
|
283
|
+
return false if flag == "0"
|
|
284
|
+
|
|
285
|
+
rails_env == "test"
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
def guard_live_call!(vendor, host:)
|
|
289
|
+
return unless live_calls_blocked?
|
|
290
|
+
return if %w[127.0.0.1 localhost ::1].include?(host.to_s)
|
|
291
|
+
|
|
292
|
+
raise LiveCallAttempted, "A live #{vendor} call was attempted with #{NO_LIVE_CALLS_ENV} armed. " \
|
|
293
|
+
"Tests drive the mock transcriber (Studio.voice_transcriber = \"mock\")."
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
# "audio/webm;codecs=opus" -> "audio/webm"; nil for a type a take cannot be.
|
|
297
|
+
def mime_type(raw)
|
|
298
|
+
base = raw.to_s.split(";").first.to_s.strip.downcase
|
|
299
|
+
EXTENSIONS.key?(base) ? base : nil
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
def filename_for(mime)
|
|
303
|
+
"take.#{EXTENSIONS.fetch(mime_type(mime) || 'audio/webm')}"
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
# Why `text` cannot be what someone said, or nil when it can (Voicebox's
|
|
307
|
+
# RecordingPipeline.no_speech_reason): a model handed near-silence writes a
|
|
308
|
+
# stock phrase, a filler, or a word in another alphabet.
|
|
309
|
+
def no_speech_reason(text, language: Studio.voice_transcribe_language)
|
|
310
|
+
text = text.to_s.strip
|
|
311
|
+
return "the transcript was empty" if text.empty?
|
|
312
|
+
if LATIN_LANGUAGES.include?(language.to_s) && !text.match?(/\p{Latin}/)
|
|
313
|
+
return "the transcript has no #{language} letters, so the model was guessing"
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
words = text.scan(/[\p{L}\p{N}']+/)
|
|
317
|
+
return "the transcript has no words" if words.empty?
|
|
318
|
+
return "the transcript is a stock phrase speech models write for silence" if SILENCE_PHRASES.include?(words.join(" ").downcase)
|
|
319
|
+
return "the transcript is only a filler (#{words.first})" if words.size == 1 && FILLERS.include?(words.first.downcase)
|
|
320
|
+
|
|
321
|
+
nil
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# One take, understood. audio: an IO (or String) of the take's bytes;
|
|
325
|
+
# mime_type: the recorder's mimeType; user: the signed-in user (or nil);
|
|
326
|
+
# embedding: the take's speaker vector from the browser (Array or JSON
|
|
327
|
+
# String), model: the embedder id that made it. Raises NotConfigured,
|
|
328
|
+
# HttpError, ArgumentError (a type a take cannot be); never logs the audio
|
|
329
|
+
# or the vector.
|
|
330
|
+
def hear(audio:, mime_type:, user:, embedding: nil, model: nil, transcriber: self.transcriber)
|
|
331
|
+
mime = self.mime_type(mime_type)
|
|
332
|
+
raise ArgumentError, "unsupported audio type #{mime_type.to_s[0, 60].inspect}" if mime.nil?
|
|
333
|
+
|
|
334
|
+
io = audio.is_a?(String) ? StringIO.new(audio) : audio
|
|
335
|
+
text = transcriber.transcribe(io: io, filename: filename_for(mime), content_type: mime).to_s.strip
|
|
336
|
+
speaker, score, check = who_spoke(user, embedding, model)
|
|
337
|
+
Heard.new(text: text, speaker: speaker, score: score, identified: check == "matched", check: check)
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# [speaker label, score or nil, check] for one take against the user's own
|
|
341
|
+
# saved voice. Same embedder revision only: the take's model must be the
|
|
342
|
+
# app's embedder (Studio.voice_embedder) AND the one the user's voice was
|
|
343
|
+
# saved with.
|
|
344
|
+
def who_spoke(user, embedding, model)
|
|
345
|
+
embedder = Studio.voice_embedder.to_s
|
|
346
|
+
probe = embedding.nil? ? nil : Studio::VoiceMatch.parse(embedding, size: Studio::VoiceMatch.size(embedder))
|
|
347
|
+
probe = nil if probe && Studio::VoiceMatch.normalize(probe).nil?
|
|
348
|
+
return [SOMEONE_NEW, nil, "no_embedding"] if probe.nil?
|
|
349
|
+
|
|
350
|
+
saved_model = user.respond_to?(:voice_embedding_model) ? user.voice_embedding_model.to_s : ""
|
|
351
|
+
saved = user.respond_to?(:voice_embedding) ? Studio::VoiceMatch.vector(user.voice_embedding) : nil
|
|
352
|
+
return [SOMEONE_NEW, nil, "no_profile"] if saved.nil? || saved_model.empty?
|
|
353
|
+
|
|
354
|
+
app_model = Studio::VoiceMatch.model_id(embedder)
|
|
355
|
+
return [SOMEONE_NEW, nil, "embedder_mismatch"] unless model.to_s == app_model && saved_model == app_model
|
|
356
|
+
|
|
357
|
+
result = Studio::VoiceMatch.match(probe, among: [user], model: app_model)
|
|
358
|
+
return [SOMEONE_NEW, nil, "embedder_mismatch"] if result.score.nil?
|
|
359
|
+
return [SOMEONE_NEW, result.score, "below_threshold"] if result.unknown?
|
|
360
|
+
|
|
361
|
+
[speaker_name(user), result.score, "matched"]
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
def speaker_name(user)
|
|
365
|
+
%i[display_name name email].each do |attr|
|
|
366
|
+
next unless user.respond_to?(attr)
|
|
367
|
+
|
|
368
|
+
value = user.public_send(attr).to_s.strip
|
|
369
|
+
return value unless value.empty?
|
|
370
|
+
end
|
|
371
|
+
"You"
|
|
372
|
+
end
|
|
373
|
+
|
|
374
|
+
# A multipart/form-data body. fields: [name, value] or [name, io, {filename:, content_type:}].
|
|
375
|
+
def multipart(fields, boundary)
|
|
376
|
+
body = +"".b
|
|
377
|
+
fields.each do |name, value, file|
|
|
378
|
+
body << "--#{boundary}\r\n".b
|
|
379
|
+
if file
|
|
380
|
+
body << %(Content-Disposition: form-data; name="#{name}"; filename="#{file[:filename]}"\r\n).b
|
|
381
|
+
body << "Content-Type: #{file[:content_type]}\r\n\r\n".b
|
|
382
|
+
body << value.read.to_s.b
|
|
383
|
+
else
|
|
384
|
+
body << %(Content-Disposition: form-data; name="#{name}"\r\n\r\n).b
|
|
385
|
+
body << value.to_s.b
|
|
386
|
+
end
|
|
387
|
+
body << "\r\n".b
|
|
388
|
+
end
|
|
389
|
+
body << "--#{boundary}--\r\n".b
|
|
390
|
+
body
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
def rails_env
|
|
394
|
+
return nil unless defined?(::Rails) && ::Rails.respond_to?(:env)
|
|
395
|
+
|
|
396
|
+
::Rails.env.to_s
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
def log_warn(message)
|
|
400
|
+
logger = defined?(::Rails) && ::Rails.respond_to?(:logger) ? ::Rails.logger : nil
|
|
401
|
+
logger&.warn("[studio][voice] #{message}")
|
|
402
|
+
end
|
|
403
|
+
end
|
|
404
|
+
end
|
data/lib/studio.rb
CHANGED
|
@@ -46,6 +46,9 @@ require "studio/loader"
|
|
|
46
46
|
require "studio/auth_labels"
|
|
47
47
|
require "studio/profile_sections"
|
|
48
48
|
require "studio/profile_image"
|
|
49
|
+
require "studio/voice_match"
|
|
50
|
+
require "studio/voice_capture"
|
|
51
|
+
require "studio/voice_transcription"
|
|
49
52
|
require "studio/oauth_identity"
|
|
50
53
|
require "studio/google_sign_in"
|
|
51
54
|
require "studio/newsletter"
|
|
@@ -556,6 +559,46 @@ module Studio
|
|
|
556
559
|
# wants the page gone still sets this false.
|
|
557
560
|
mattr_accessor :draw_profile_routes, default: true
|
|
558
561
|
|
|
562
|
+
# The voice row on /profile: "Set up your voice". The signed-in user reads
|
|
563
|
+
# three short phrases; each take becomes a speaker embedding ON THEIR DEVICE
|
|
564
|
+
# (studio/voice_profile, a WeSpeaker model run by transformers.js), the audio
|
|
565
|
+
# is dropped, and only the vectors are posted. The server averages them into
|
|
566
|
+
# one vector on the user's own record (users.voice_embedding, installed by the
|
|
567
|
+
# engine migration add_voice_profile_to_users). Studio::VoiceMatch.identify
|
|
568
|
+
# answers "who is speaking" from an embedding.
|
|
569
|
+
#
|
|
570
|
+
# OFF BY DEFAULT. Turning it on is a privacy decision (a voice pattern is
|
|
571
|
+
# biometric data in several jurisdictions) and it makes the page fetch a model
|
|
572
|
+
# from Hugging Face and its runtime from jsDelivr, which a host's CSP and
|
|
573
|
+
# privacy page must both name. docs/VOICE_PROFILE.md lists what to change.
|
|
574
|
+
mattr_accessor :voice_profile, default: false
|
|
575
|
+
|
|
576
|
+
# Which speaker embedder the browser runs: "wespeaker" (the real one) or
|
|
577
|
+
# "fake" (tests only; a fixed vector per window.studioFakeVoice). The server
|
|
578
|
+
# refuses a voice saved by any other embedder, so a page cannot pick its own.
|
|
579
|
+
mattr_accessor :voice_embedder, default: "wespeaker"
|
|
580
|
+
|
|
581
|
+
# Understand (docs/VOICE_TRANSCRIPTION.md): POST /voice/takes turns a signed-in
|
|
582
|
+
# user's take into {text, speaker, score}. OFF BY DEFAULT: every take costs a
|
|
583
|
+
# speech-to-text call, so an app turns it on when a page posts takes.
|
|
584
|
+
mattr_accessor :voice_transcription, default: false
|
|
585
|
+
|
|
586
|
+
# Which transcriber answers: "openai" or "mock". nil means
|
|
587
|
+
# STUDIO_VOICE_TRANSCRIBER, else "openai" in production and "mock" elsewhere
|
|
588
|
+
# (Studio::VoiceTranscription.adapter_name). Tests never reach the vendor:
|
|
589
|
+
# VOICE_NO_LIVE_CALLS is armed in every test environment.
|
|
590
|
+
mattr_accessor :voice_transcriber, default: nil
|
|
591
|
+
|
|
592
|
+
# The OpenAI key for the real transcriber. nil reads OPENAI_API_KEY. Missing,
|
|
593
|
+
# the endpoint answers 503 with a line the page can show; nothing crashes.
|
|
594
|
+
mattr_accessor :voice_openai_api_key, default: nil
|
|
595
|
+
|
|
596
|
+
# The speech-to-text models (Voicebox's measured pair) and the language both
|
|
597
|
+
# are told to hear. A blank fallback turns the one retry off.
|
|
598
|
+
mattr_accessor :voice_transcribe_model, default: "gpt-4o-transcribe"
|
|
599
|
+
mattr_accessor :voice_transcribe_fallback_model, default: "whisper-1"
|
|
600
|
+
mattr_accessor :voice_transcribe_language, default: "en"
|
|
601
|
+
|
|
559
602
|
# How long a freshly minted magic link stays live.
|
|
560
603
|
mattr_accessor :magic_link_ttl, default: 15.minutes
|
|
561
604
|
|
|
@@ -1926,6 +1969,20 @@ module Studio
|
|
|
1926
1969
|
post "profile/newsletter", to: "studio/profiles#subscribe_newsletter",
|
|
1927
1970
|
as: :profile_newsletter
|
|
1928
1971
|
delete "profile/newsletter", to: "studio/profiles#unsubscribe_newsletter"
|
|
1972
|
+
|
|
1973
|
+
# The voice row. POST saves (or replaces) the voice from the setup's
|
|
1974
|
+
# vectors; DELETE removes it. Drawn with the page, like the newsletter
|
|
1975
|
+
# pair; the actions refuse unless the row renders (Studio.voice_profile
|
|
1976
|
+
# on, and the user has the columns).
|
|
1977
|
+
post "profile/voice", to: "studio/profiles#save_voice", as: :profile_voice
|
|
1978
|
+
delete "profile/voice", to: "studio/profiles#delete_voice"
|
|
1979
|
+
end
|
|
1980
|
+
|
|
1981
|
+
# Understand: one signed-in take in, {text, speaker, score} out
|
|
1982
|
+
# (Studio::VoiceTakesController, docs/VOICE_TRANSCRIPTION.md). Drawn only
|
|
1983
|
+
# for an app that turns it on, because every take is a paid vendor call.
|
|
1984
|
+
if Studio.voice_transcription
|
|
1985
|
+
post "voice/takes", to: "studio/voice_takes#create", as: :studio_voice_takes
|
|
1929
1986
|
end
|
|
1930
1987
|
|
|
1931
1988
|
resources :error_logs, only: [:index, :show]
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: studio-engine
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.121.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Alex McRitchie
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-11-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rails
|
|
@@ -473,6 +473,7 @@ files:
|
|
|
473
473
|
- app/controllers/studio/session_states_controller.rb
|
|
474
474
|
- app/controllers/studio/site_identities_controller.rb
|
|
475
475
|
- app/controllers/studio/surveys_controller.rb
|
|
476
|
+
- app/controllers/studio/voice_takes_controller.rb
|
|
476
477
|
- app/controllers/style_controller.rb
|
|
477
478
|
- app/controllers/theme_settings_controller.rb
|
|
478
479
|
- app/helpers/studio/admin_models_table_helper.rb
|
|
@@ -507,11 +508,14 @@ files:
|
|
|
507
508
|
- app/javascript/studio/controllers/knowledge_transcript_controller.js
|
|
508
509
|
- app/javascript/studio/controllers/link_preview_card_controller.js
|
|
509
510
|
- app/javascript/studio/controllers/link_sidebar_controller.js
|
|
511
|
+
- app/javascript/studio/controllers/microphone_chip_controller.js
|
|
510
512
|
- app/javascript/studio/controllers/modal_host_controller.js
|
|
511
513
|
- app/javascript/studio/controllers/nav_collapse_controller.js
|
|
512
514
|
- app/javascript/studio/controllers/style_modals_controller.js
|
|
513
515
|
- app/javascript/studio/controllers/survey_controller.js
|
|
514
516
|
- app/javascript/studio/controllers/toast_controller.js
|
|
517
|
+
- app/javascript/studio/controllers/voice_profile_controller.js
|
|
518
|
+
- app/javascript/studio/controllers/voice_recorder_controller.js
|
|
515
519
|
- app/javascript/studio/cropper.js
|
|
516
520
|
- app/javascript/studio/email_banner.js
|
|
517
521
|
- app/javascript/studio/email_banner_scale.js
|
|
@@ -525,11 +529,14 @@ files:
|
|
|
525
529
|
- app/javascript/studio/image_upload.js
|
|
526
530
|
- app/javascript/studio/knowledge_transcript.js
|
|
527
531
|
- app/javascript/studio/lazy_controllers.js
|
|
532
|
+
- app/javascript/studio/level_meter.js
|
|
528
533
|
- app/javascript/studio/leveling_activity.js
|
|
529
534
|
- app/javascript/studio/link_preview_card.js
|
|
530
535
|
- app/javascript/studio/link_sidebar.js
|
|
531
536
|
- app/javascript/studio/loader.js
|
|
532
537
|
- app/javascript/studio/local_path.js
|
|
538
|
+
- app/javascript/studio/microphone.js
|
|
539
|
+
- app/javascript/studio/microphone_chip.js
|
|
533
540
|
- app/javascript/studio/modal_countdown.js
|
|
534
541
|
- app/javascript/studio/modal_host.js
|
|
535
542
|
- app/javascript/studio/nav_collapse.js
|
|
@@ -545,6 +552,8 @@ files:
|
|
|
545
552
|
- app/javascript/studio/theme_editor.js
|
|
546
553
|
- app/javascript/studio/toast.js
|
|
547
554
|
- app/javascript/studio/vendor/stimulus.js
|
|
555
|
+
- app/javascript/studio/voice_profile.js
|
|
556
|
+
- app/javascript/studio/voice_recorder.js
|
|
548
557
|
- app/jobs/error_log_cleanup_job.rb
|
|
549
558
|
- app/jobs/studio/email_delivery_job.rb
|
|
550
559
|
- app/jobs/studio/health_report_job.rb
|
|
@@ -738,6 +747,7 @@ files:
|
|
|
738
747
|
- app/views/studio/profiles/_pencil_icon.html.erb
|
|
739
748
|
- app/views/studio/profiles/_plain_save.html.erb
|
|
740
749
|
- app/views/studio/profiles/_save_controls.html.erb
|
|
750
|
+
- app/views/studio/profiles/_voice_section.html.erb
|
|
741
751
|
- app/views/studio/profiles/edit.html.erb
|
|
742
752
|
- app/views/studio/profiles/show.html.erb
|
|
743
753
|
- app/views/studio/public_users/not_found.html.erb
|
|
@@ -756,6 +766,10 @@ files:
|
|
|
756
766
|
- app/views/studio/surveys/show.html.erb
|
|
757
767
|
- app/views/studio/surveys/sign_in_required.html.erb
|
|
758
768
|
- app/views/studio/surveys/thanks.html.erb
|
|
769
|
+
- app/views/studio/voice/_microphone_chip.html.erb
|
|
770
|
+
- app/views/studio/voice/_microphone_help.html.erb
|
|
771
|
+
- app/views/studio/voice/_recorder.html.erb
|
|
772
|
+
- app/views/studio/voice/_styles.html.erb
|
|
759
773
|
- app/views/style/_components.html.erb
|
|
760
774
|
- app/views/style/_host.html.erb
|
|
761
775
|
- app/views/style/_modal_specimen.html.erb
|
|
@@ -806,6 +820,7 @@ files:
|
|
|
806
820
|
- db/migrate/20261006120003_ensure_image_caches_table.rb
|
|
807
821
|
- db/migrate/20261008120000_add_recording_to_studio_knowledge_docs.rb
|
|
808
822
|
- db/migrate/20261010120000_add_stored_in_to_image_caches.rb
|
|
823
|
+
- db/migrate/20261029120000_add_voice_profile_to_users.rb
|
|
809
824
|
- docs/BOOKING.md
|
|
810
825
|
- docs/SITE_FOOTER.md
|
|
811
826
|
- lib/active_storage/service/studio_trash_s3_service.rb
|
|
@@ -889,6 +904,9 @@ files:
|
|
|
889
904
|
- lib/studio/ui_primitives.rb
|
|
890
905
|
- lib/studio/username_generator.rb
|
|
891
906
|
- lib/studio/version.rb
|
|
907
|
+
- lib/studio/voice_capture.rb
|
|
908
|
+
- lib/studio/voice_match.rb
|
|
909
|
+
- lib/studio/voice_transcription.rb
|
|
892
910
|
- lib/tasks/studio_email.rake
|
|
893
911
|
- lib/tasks/studio_knowledge.rake
|
|
894
912
|
- lib/tasks/studio_node.rake
|