studio-engine 0.120.0 → 0.122.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +84 -0
- data/README.md +2 -0
- data/app/controllers/studio/voice_takes_controller.rb +146 -0
- data/app/javascript/studio/controllers/microphone_chip_controller.js +21 -0
- data/app/javascript/studio/controllers/voice_recorder_controller.js +29 -0
- data/app/javascript/studio/level_meter.js +167 -0
- data/app/javascript/studio/microphone.js +250 -0
- data/app/javascript/studio/microphone_chip.js +374 -0
- data/app/javascript/studio/stimulus.js +6 -1
- data/app/javascript/studio/voice_profile.js +38 -113
- data/app/javascript/studio/voice_recorder.js +363 -0
- data/app/views/studio/voice/_microphone_chip.html.erb +45 -0
- data/app/views/studio/voice/_microphone_help.html.erb +65 -0
- data/app/views/studio/voice/_recorder.html.erb +34 -0
- data/app/views/studio/voice/_styles.html.erb +94 -0
- data/lib/studio/engine.rb +3 -1
- data/lib/studio/version.rb +1 -1
- data/lib/studio/voice_capture.rb +95 -0
- data/lib/studio/voice_transcription.rb +470 -0
- data/lib/studio.rb +45 -0
- metadata +15 -2
|
@@ -0,0 +1,470 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "net/http"
|
|
5
|
+
require "openssl"
|
|
6
|
+
require "securerandom"
|
|
7
|
+
require "stringio"
|
|
8
|
+
require "timeout"
|
|
9
|
+
require "uri"
|
|
10
|
+
|
|
11
|
+
# Understand: one finished take (the recorder's blob and its mime type, plus the
|
|
12
|
+
# speaker embedding the browser made of it) becomes {text, speaker, score}.
|
|
13
|
+
#
|
|
14
|
+
# The server half of the voice layer (docs/VOICE_TRANSCRIPTION.md). The capture
|
|
15
|
+
# half (docs/VOICE_CAPTURE.md) hands the page a take; the page posts it here,
|
|
16
|
+
# through Studio::VoiceTakesController (POST /voice/takes) or by calling
|
|
17
|
+
# Studio::VoiceTranscription.hear itself.
|
|
18
|
+
#
|
|
19
|
+
# heard = Studio::VoiceTranscription.hear(audio: io, mime_type: "audio/webm",
|
|
20
|
+
# user: current_user,
|
|
21
|
+
# embedding: vector, model: model_id)
|
|
22
|
+
# heard.to_h # => {"text"=>"...", "speaker"=>"Pat Studio", "score"=>0.88,
|
|
23
|
+
# # "identified"=>true, "check"=>"matched"}
|
|
24
|
+
#
|
|
25
|
+
# MOCK-FIRST, as Voicebox runs it (config/initializers/voicebox.rb, Adapters):
|
|
26
|
+
# the transcriber is chosen by name, production defaults to the real one and
|
|
27
|
+
# everywhere else to the mock, and a real adapter raises LiveCallAttempted
|
|
28
|
+
# before it opens a socket while VOICE_NO_LIVE_CALLS is armed (always, in a test
|
|
29
|
+
# environment). Ported from Voicebox's Transcription::OpenAI and
|
|
30
|
+
# Transcription::Mock.
|
|
31
|
+
#
|
|
32
|
+
# WHO SPOKE is Studio::VoiceMatch against the SIGNED-IN USER'S OWN saved voice
|
|
33
|
+
# (users.voice_embedding), compared only when both vectors came from the same
|
|
34
|
+
# embedder revision. Named people without an account are a later card.
|
|
35
|
+
#
|
|
36
|
+
# NOTHING IS KEPT. The audio is read once, sent to the transcriber and dropped;
|
|
37
|
+
# no vector or audio is logged (the engine's parameter filter masks
|
|
38
|
+
# voice[embedding] and voice[audio]). Pure Ruby apart from the optional Rails
|
|
39
|
+
# logger and environment, so the unit suite runs it without an app.
|
|
40
|
+
module Studio
|
|
41
|
+
module VoiceTranscription
|
|
42
|
+
# A transcriber call that failed.
|
|
43
|
+
class Error < StandardError; end
|
|
44
|
+
|
|
45
|
+
# The vendor failed. The status rides along (0 when nothing answered) with a
|
|
46
|
+
# reason code, so a caller can tell an auth failure from a model that fell
|
|
47
|
+
# over. The message names only the model, the status and the reason, never
|
|
48
|
+
# the vendor's own words: it reaches ErrorLog, and a vendor's error text can
|
|
49
|
+
# echo what it was sent.
|
|
50
|
+
class HttpError < Error
|
|
51
|
+
# Why, as a code: unauthorized (401/403), rate_limited (429),
|
|
52
|
+
# server_error (5xx), rejected (another non-2xx), timeout (the deadline
|
|
53
|
+
# ran out), unreachable (no answer: refused, reset, DNS, TLS).
|
|
54
|
+
REASONS = %w[unauthorized rate_limited server_error rejected timeout unreachable].freeze
|
|
55
|
+
|
|
56
|
+
attr_reader :status, :reason
|
|
57
|
+
|
|
58
|
+
def initialize(message, status:, reason: nil)
|
|
59
|
+
super(message)
|
|
60
|
+
@status = status
|
|
61
|
+
@reason = reason || self.class.reason_for(status)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def self.reason_for(status)
|
|
65
|
+
case status.to_i
|
|
66
|
+
when 401, 403 then "unauthorized"
|
|
67
|
+
when 429 then "rate_limited"
|
|
68
|
+
when 500..599 then "server_error"
|
|
69
|
+
when 0 then "unreachable"
|
|
70
|
+
else "rejected"
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# A real transcriber chosen without its credential, or a name that is no
|
|
76
|
+
# adapter. The page shows NOT_CONFIGURED_MESSAGE; the cause is in the log.
|
|
77
|
+
class NotConfigured < Error; end
|
|
78
|
+
|
|
79
|
+
# DELIBERATELY OUTSIDE StandardError (Voicebox's Adapters::LiveCallAttempted):
|
|
80
|
+
# every caller turns a StandardError into a friendly failure, so a trap that
|
|
81
|
+
# was one would be swallowed by the code it guards and a test that reached
|
|
82
|
+
# a real vendor would pass quietly.
|
|
83
|
+
class LiveCallAttempted < Exception; end # rubocop:disable Lint/InheritException
|
|
84
|
+
|
|
85
|
+
NO_LIVE_CALLS_ENV = "VOICE_NO_LIVE_CALLS"
|
|
86
|
+
TRANSCRIBER_ENV = "STUDIO_VOICE_TRANSCRIBER"
|
|
87
|
+
# Voice's own key. Deliberately NOT the app's shared OPENAI_API_KEY.
|
|
88
|
+
API_KEY_ENV = "STUDIO_VOICE_OPENAI_API_KEY"
|
|
89
|
+
|
|
90
|
+
# What a page shows when the app has no transcriber key. Says what is wrong
|
|
91
|
+
# without naming the variable to a visitor.
|
|
92
|
+
NOT_CONFIGURED_MESSAGE = "Voice isn't set up on this app yet, so I can't hear recordings. Type instead for now."
|
|
93
|
+
|
|
94
|
+
# The bounds on one take. The recorder stops itself at 60 seconds by default
|
|
95
|
+
# (studio/voice_recorder max_ms); a minute of opus is about half a megabyte,
|
|
96
|
+
# so 10 MB is far above any honest take and well under the vendor's 25 MB.
|
|
97
|
+
MAX_BYTES = 10 * 1024 * 1024
|
|
98
|
+
MAX_DURATION_MS = 120_000
|
|
99
|
+
|
|
100
|
+
# The audio types a take may be, as MediaRecorder names them (without the
|
|
101
|
+
# ";codecs=" parameter), and the extension the vendor reads the format from.
|
|
102
|
+
EXTENSIONS = {
|
|
103
|
+
"audio/webm" => "webm",
|
|
104
|
+
"video/webm" => "webm",
|
|
105
|
+
"audio/ogg" => "ogg",
|
|
106
|
+
"audio/mp4" => "mp4",
|
|
107
|
+
"audio/m4a" => "m4a",
|
|
108
|
+
"audio/x-m4a" => "m4a",
|
|
109
|
+
"audio/mpeg" => "mp3",
|
|
110
|
+
"audio/wav" => "wav",
|
|
111
|
+
"audio/x-wav" => "wav"
|
|
112
|
+
}.freeze
|
|
113
|
+
|
|
114
|
+
# The label a take gets when it is not the signed-in user's saved voice. The
|
|
115
|
+
# agent is told, and the message still sends (epic default).
|
|
116
|
+
SOMEONE_NEW = "Someone new"
|
|
117
|
+
|
|
118
|
+
# Why the speaker is (or is not) named, in Heard#check:
|
|
119
|
+
# matched the user's own voice, at or above VoiceMatch::THRESHOLD
|
|
120
|
+
# below_threshold compared, and the closest score fell short
|
|
121
|
+
# no_profile the user has no saved voice to compare with
|
|
122
|
+
# embedder_mismatch the take and the saved voice came from different
|
|
123
|
+
# embedders or revisions, so they were not compared
|
|
124
|
+
# no_embedding the take carried no usable speaker embedding
|
|
125
|
+
CHECKS = %w[matched below_threshold no_profile embedder_mismatch no_embedding].freeze
|
|
126
|
+
|
|
127
|
+
# Languages written in the Latin alphabet (Voicebox's RecordingPipeline).
|
|
128
|
+
LATIN_LANGUAGES = %w[en es fr de it pt nl].freeze
|
|
129
|
+
# What speech models write for silence, whole, lowercased, unpunctuated.
|
|
130
|
+
SILENCE_PHRASES = [
|
|
131
|
+
"thank you", "thanks", "thank you for watching", "thanks for watching",
|
|
132
|
+
"thank you for listening", "thanks for listening", "bye", "bye bye", "you"
|
|
133
|
+
].freeze
|
|
134
|
+
FILLERS = %w[um umm uh uhh hmm mm mhm ah er erm].freeze
|
|
135
|
+
|
|
136
|
+
Heard = Struct.new(:text, :speaker, :score, :identified, :check, keyword_init: true) do
|
|
137
|
+
def identified? = identified
|
|
138
|
+
|
|
139
|
+
def to_h
|
|
140
|
+
{ "text" => text, "speaker" => speaker, "score" => score, "identified" => identified, "check" => check }
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# The real speech-to-text: OpenAI's POST /v1/audio/transcriptions, one
|
|
145
|
+
# multipart file. Ported from Voicebox's Transcription::OpenAI: the model is
|
|
146
|
+
# told the language (left to guess, a model handed near-silence answers a
|
|
147
|
+
# stock word in any language), and a model that fell over (a 5xx, or a 4xx
|
|
148
|
+
# other than auth and 429) is retried once on the fallback from the start of
|
|
149
|
+
# the audio.
|
|
150
|
+
#
|
|
151
|
+
# ONE DEADLINE for the whole take (Studio.voice_transcribe_deadline, 25 s):
|
|
152
|
+
# both attempts share it, every socket timeout is what is left of it, and a
|
|
153
|
+
# hard Timeout backs it, so a vendor that trickles bytes cannot hold a web
|
|
154
|
+
# thread past it. Nothing is retried after a timeout (the deadline is spent),
|
|
155
|
+
# a 429 (the account is throttled; a second model only spends more), an auth
|
|
156
|
+
# failure, or no answer at all; nor with under MIN_RETRY_SECONDS left.
|
|
157
|
+
class OpenAI
|
|
158
|
+
ENDPOINT = "https://api.openai.com/v1/audio/transcriptions"
|
|
159
|
+
# The cap on connecting, inside the deadline.
|
|
160
|
+
OPEN_TIMEOUT = 10
|
|
161
|
+
NO_RETRY = %w[unauthorized rate_limited timeout unreachable].freeze
|
|
162
|
+
MIN_RETRY_SECONDS = 2
|
|
163
|
+
|
|
164
|
+
# The hard backstop's exception; turned into HttpError(reason: "timeout").
|
|
165
|
+
class DeadlineExceeded < Timeout::Error; end
|
|
166
|
+
|
|
167
|
+
attr_reader :model, :fallback_model, :language, :endpoint, :deadline
|
|
168
|
+
|
|
169
|
+
def initialize(api_key: VoiceTranscription.api_key, model: Studio.voice_transcribe_model,
|
|
170
|
+
fallback_model: Studio.voice_transcribe_fallback_model,
|
|
171
|
+
language: Studio.voice_transcribe_language, endpoint: ENDPOINT,
|
|
172
|
+
deadline: Studio.voice_transcribe_deadline)
|
|
173
|
+
@api_key = api_key
|
|
174
|
+
@model = model.to_s
|
|
175
|
+
@fallback_model = fallback_model.to_s.strip.empty? ? nil : fallback_model.to_s
|
|
176
|
+
@language = language.to_s.strip.empty? ? nil : language.to_s
|
|
177
|
+
@endpoint = endpoint
|
|
178
|
+
@deadline = deadline.to_f.positive? ? deadline.to_f : 25.0
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def configured? = !@api_key.to_s.strip.empty?
|
|
182
|
+
|
|
183
|
+
# io: the audio, at its start. Returns the transcript, stripped.
|
|
184
|
+
def transcribe(io:, filename:, content_type:)
|
|
185
|
+
raise NotConfigured, "#{API_KEY_ENV} is not set, so this app cannot transcribe voice" unless configured?
|
|
186
|
+
|
|
187
|
+
ends_at = now + deadline
|
|
188
|
+
begin
|
|
189
|
+
request_text(model, io, filename, content_type, ends_at)
|
|
190
|
+
rescue HttpError => e
|
|
191
|
+
raise unless retry?(e, ends_at)
|
|
192
|
+
|
|
193
|
+
VoiceTranscription.log_warn("#{model} transcription failed (HTTP #{e.status}, #{e.reason}); " \
|
|
194
|
+
"retrying on #{fallback_model}")
|
|
195
|
+
io.rewind
|
|
196
|
+
request_text(fallback_model, io, filename, content_type, ends_at)
|
|
197
|
+
end
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def retry?(error, ends_at)
|
|
201
|
+
return false if fallback_model.nil? || fallback_model == model
|
|
202
|
+
return false if NO_RETRY.include?(error.reason)
|
|
203
|
+
|
|
204
|
+
ends_at - now >= MIN_RETRY_SECONDS
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# The multipart fields, apart from the socket so a test reads them.
|
|
208
|
+
def form_fields(model_name, io, filename, content_type)
|
|
209
|
+
fields = [
|
|
210
|
+
["file", io, { filename: filename, content_type: content_type }],
|
|
211
|
+
["model", model_name],
|
|
212
|
+
["response_format", "json"]
|
|
213
|
+
]
|
|
214
|
+
fields << ["language", language] if language
|
|
215
|
+
fields
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# The request, built but not sent: a multipart body with a Content-Length
|
|
219
|
+
# (not chunked), the bearer key, JSON accepted.
|
|
220
|
+
def build_request(model_name, io, filename, content_type)
|
|
221
|
+
uri = URI.parse(endpoint)
|
|
222
|
+
boundary = "studio-voice-#{SecureRandom.hex(12)}"
|
|
223
|
+
request = Net::HTTP::Post.new(uri)
|
|
224
|
+
request["Authorization"] = "Bearer #{@api_key}"
|
|
225
|
+
request["Accept"] = "application/json"
|
|
226
|
+
request["Content-Type"] = "multipart/form-data; boundary=#{boundary}"
|
|
227
|
+
request.body = VoiceTranscription.multipart(form_fields(model_name, io, filename, content_type), boundary)
|
|
228
|
+
request
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
private
|
|
232
|
+
|
|
233
|
+
def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
234
|
+
|
|
235
|
+
def request_text(model_name, io, filename, content_type, ends_at)
|
|
236
|
+
perform(model_name, io, filename, content_type, ends_at).fetch("text", "").to_s.strip
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# The only place a socket opens, so the only place the trap sits.
|
|
240
|
+
# uri.hostname, not uri.host: an IPv6 literal's host keeps its brackets
|
|
241
|
+
# ("[::1]"), which neither the loopback check nor the connect reads.
|
|
242
|
+
def perform(model_name, io, filename, content_type, ends_at)
|
|
243
|
+
uri = URI.parse(endpoint)
|
|
244
|
+
VoiceTranscription.guard_live_call!("OpenAI", host: uri.hostname)
|
|
245
|
+
|
|
246
|
+
left = ends_at - now
|
|
247
|
+
raise HttpError.new("OpenAI #{model_name} was not tried: the take's deadline ran out", status: 0, reason: "timeout") if left <= 0
|
|
248
|
+
|
|
249
|
+
request = build_request(model_name, io, filename, content_type)
|
|
250
|
+
response = Timeout.timeout(left, DeadlineExceeded) do
|
|
251
|
+
Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https", open_timeout: [OPEN_TIMEOUT, left].min,
|
|
252
|
+
read_timeout: left, write_timeout: left) do |http|
|
|
253
|
+
http.request(request)
|
|
254
|
+
end
|
|
255
|
+
end
|
|
256
|
+
interpret(response, model_name)
|
|
257
|
+
rescue Timeout::Error => e
|
|
258
|
+
raise HttpError.new("OpenAI #{model_name} did not answer within the take's #{deadline.round(1)} s deadline (#{e.class})",
|
|
259
|
+
status: 0, reason: "timeout")
|
|
260
|
+
rescue SystemCallError, IOError, OpenSSL::SSL::SSLError, SocketError => e
|
|
261
|
+
raise HttpError.new("OpenAI #{model_name} could not be reached (#{e.class})", status: 0, reason: "unreachable")
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# A failure carries the status and its reason code, never the vendor's
|
|
265
|
+
# words (HttpError).
|
|
266
|
+
def interpret(response, model_name)
|
|
267
|
+
status = response.code.to_i
|
|
268
|
+
unless response.is_a?(Net::HTTPSuccess)
|
|
269
|
+
reason = HttpError.reason_for(status)
|
|
270
|
+
raise HttpError.new("OpenAI #{model_name} answered HTTP #{status} (#{reason})", status: status, reason: reason)
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
parsed = begin
|
|
274
|
+
JSON.parse(response.body.to_s)
|
|
275
|
+
rescue JSON::ParserError
|
|
276
|
+
{}
|
|
277
|
+
end
|
|
278
|
+
parsed.is_a?(Hash) ? parsed : {}
|
|
279
|
+
end
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# The default everywhere but production: no network, a fixed sentence. A
|
|
283
|
+
# test swaps the answer with Mock.reply = ->(**) { "..." } and resets with
|
|
284
|
+
# Mock.reset!. It records what it was handed (sizes, never the bytes).
|
|
285
|
+
class Mock
|
|
286
|
+
DEFAULT_TEXT = "This is a mock transcript of your voice."
|
|
287
|
+
|
|
288
|
+
class << self
|
|
289
|
+
attr_accessor :reply
|
|
290
|
+
attr_writer :calls
|
|
291
|
+
|
|
292
|
+
def calls = (@calls ||= [])
|
|
293
|
+
|
|
294
|
+
def reset!
|
|
295
|
+
self.reply = nil
|
|
296
|
+
self.calls = []
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def configured? = true
|
|
301
|
+
|
|
302
|
+
def transcribe(io:, filename:, content_type:)
|
|
303
|
+
bytes = io.read.to_s.bytesize
|
|
304
|
+
self.class.calls << { filename: filename, content_type: content_type, bytes: bytes }
|
|
305
|
+
reply = self.class.reply
|
|
306
|
+
reply.nil? ? DEFAULT_TEXT : reply.call(filename: filename, content_type: content_type, bytes: bytes).to_s
|
|
307
|
+
end
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
ADAPTERS = { "openai" => OpenAI, "mock" => Mock }.freeze
|
|
311
|
+
|
|
312
|
+
module_function
|
|
313
|
+
|
|
314
|
+
# The transcriber's name: Studio.voice_transcriber when the host set one,
|
|
315
|
+
# else STUDIO_VOICE_TRANSCRIBER, else "openai" in production and "mock"
|
|
316
|
+
# everywhere else.
|
|
317
|
+
def adapter_name
|
|
318
|
+
configured = Studio.voice_transcriber.to_s.strip
|
|
319
|
+
return configured unless configured.empty?
|
|
320
|
+
|
|
321
|
+
from_env = ENV[TRANSCRIBER_ENV].to_s.strip
|
|
322
|
+
return from_env unless from_env.empty?
|
|
323
|
+
|
|
324
|
+
rails_env == "production" ? "openai" : "mock"
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
def transcriber(name = adapter_name)
|
|
328
|
+
klass = ADAPTERS.fetch(name.to_s) do
|
|
329
|
+
raise NotConfigured, "Studio.voice_transcriber=#{name.inspect} names no adapter; one of: #{ADAPTERS.keys.join(', ')}"
|
|
330
|
+
end
|
|
331
|
+
klass.new
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
# Studio.voice_openai_api_key, else STUDIO_VOICE_OPENAI_API_KEY. Never the
|
|
335
|
+
# app's shared OPENAI_API_KEY: voice spends on its own key and budget.
|
|
336
|
+
def api_key
|
|
337
|
+
key = Studio.voice_openai_api_key.to_s.strip
|
|
338
|
+
key.empty? ? ENV[API_KEY_ENV].to_s.strip : key
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
# Armed when VOICE_NO_LIVE_CALLS=1, and in any Rails test environment unless
|
|
342
|
+
# it is set to 0. A call to this machine (a stub server) is not live.
|
|
343
|
+
def live_calls_blocked?
|
|
344
|
+
flag = ENV[NO_LIVE_CALLS_ENV].to_s.strip
|
|
345
|
+
return true if flag == "1"
|
|
346
|
+
return false if flag == "0"
|
|
347
|
+
|
|
348
|
+
rails_env == "test"
|
|
349
|
+
end
|
|
350
|
+
|
|
351
|
+
# This machine. host is URI#hostname, so IPv6 arrives unbracketed.
|
|
352
|
+
LOOPBACK_HOSTS = %w[127.0.0.1 localhost ::1].freeze
|
|
353
|
+
|
|
354
|
+
def guard_live_call!(vendor, host:)
|
|
355
|
+
return unless live_calls_blocked?
|
|
356
|
+
return if LOOPBACK_HOSTS.include?(host.to_s)
|
|
357
|
+
|
|
358
|
+
raise LiveCallAttempted, "A live #{vendor} call was attempted with #{NO_LIVE_CALLS_ENV} armed. " \
|
|
359
|
+
"Tests drive the mock transcriber (Studio.voice_transcriber = \"mock\")."
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
# "audio/webm;codecs=opus" -> "audio/webm"; nil for a type a take cannot be.
|
|
363
|
+
def mime_type(raw)
|
|
364
|
+
base = raw.to_s.split(";").first.to_s.strip.downcase
|
|
365
|
+
EXTENSIONS.key?(base) ? base : nil
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
def filename_for(mime)
|
|
369
|
+
"take.#{EXTENSIONS.fetch(mime_type(mime) || 'audio/webm')}"
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
# Why `text` cannot be what someone said, or nil when it can (Voicebox's
|
|
373
|
+
# RecordingPipeline.no_speech_reason): a model handed near-silence writes a
|
|
374
|
+
# stock phrase, a filler, or a word in another alphabet.
|
|
375
|
+
def no_speech_reason(text, language: Studio.voice_transcribe_language)
|
|
376
|
+
text = text.to_s.strip
|
|
377
|
+
return "the transcript was empty" if text.empty?
|
|
378
|
+
if LATIN_LANGUAGES.include?(language.to_s) && !text.match?(/\p{Latin}/)
|
|
379
|
+
return "the transcript has no #{language} letters, so the model was guessing"
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
words = text.scan(/[\p{L}\p{N}']+/)
|
|
383
|
+
return "the transcript has no words" if words.empty?
|
|
384
|
+
return "the transcript is a stock phrase speech models write for silence" if SILENCE_PHRASES.include?(words.join(" ").downcase)
|
|
385
|
+
return "the transcript is only a filler (#{words.first})" if words.size == 1 && FILLERS.include?(words.first.downcase)
|
|
386
|
+
|
|
387
|
+
nil
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
# One take, understood. audio: an IO (or String) of the take's bytes;
|
|
391
|
+
# mime_type: the recorder's mimeType; user: the signed-in user (or nil);
|
|
392
|
+
# embedding: the take's speaker vector from the browser (Array or JSON
|
|
393
|
+
# String), model: the embedder id that made it. Raises NotConfigured,
|
|
394
|
+
# HttpError, ArgumentError (a type a take cannot be); never logs the audio
|
|
395
|
+
# or the vector.
|
|
396
|
+
def hear(audio:, mime_type:, user:, embedding: nil, model: nil, transcriber: self.transcriber)
|
|
397
|
+
mime = self.mime_type(mime_type)
|
|
398
|
+
raise ArgumentError, "unsupported audio type #{mime_type.to_s[0, 60].inspect}" if mime.nil?
|
|
399
|
+
|
|
400
|
+
io = audio.is_a?(String) ? StringIO.new(audio) : audio
|
|
401
|
+
text = transcriber.transcribe(io: io, filename: filename_for(mime), content_type: mime).to_s.strip
|
|
402
|
+
speaker, score, check = who_spoke(user, embedding, model)
|
|
403
|
+
Heard.new(text: text, speaker: speaker, score: score, identified: check == "matched", check: check)
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
# [speaker label, score or nil, check] for one take against the user's own
|
|
407
|
+
# saved voice. Same embedder revision only: the take's model must be the
|
|
408
|
+
# app's embedder (Studio.voice_embedder) AND the one the user's voice was
|
|
409
|
+
# saved with.
|
|
410
|
+
def who_spoke(user, embedding, model)
|
|
411
|
+
embedder = Studio.voice_embedder.to_s
|
|
412
|
+
probe = embedding.nil? ? nil : Studio::VoiceMatch.parse(embedding, size: Studio::VoiceMatch.size(embedder))
|
|
413
|
+
probe = nil if probe && Studio::VoiceMatch.normalize(probe).nil?
|
|
414
|
+
return [SOMEONE_NEW, nil, "no_embedding"] if probe.nil?
|
|
415
|
+
|
|
416
|
+
saved_model = user.respond_to?(:voice_embedding_model) ? user.voice_embedding_model.to_s : ""
|
|
417
|
+
saved = user.respond_to?(:voice_embedding) ? Studio::VoiceMatch.vector(user.voice_embedding) : nil
|
|
418
|
+
return [SOMEONE_NEW, nil, "no_profile"] if saved.nil? || saved_model.empty?
|
|
419
|
+
|
|
420
|
+
app_model = Studio::VoiceMatch.model_id(embedder)
|
|
421
|
+
return [SOMEONE_NEW, nil, "embedder_mismatch"] unless model.to_s == app_model && saved_model == app_model
|
|
422
|
+
|
|
423
|
+
result = Studio::VoiceMatch.match(probe, among: [user], model: app_model)
|
|
424
|
+
return [SOMEONE_NEW, nil, "embedder_mismatch"] if result.score.nil?
|
|
425
|
+
return [SOMEONE_NEW, result.score, "below_threshold"] if result.unknown?
|
|
426
|
+
|
|
427
|
+
[speaker_name(user), result.score, "matched"]
|
|
428
|
+
end
|
|
429
|
+
|
|
430
|
+
def speaker_name(user)
|
|
431
|
+
%i[display_name name email].each do |attr|
|
|
432
|
+
next unless user.respond_to?(attr)
|
|
433
|
+
|
|
434
|
+
value = user.public_send(attr).to_s.strip
|
|
435
|
+
return value unless value.empty?
|
|
436
|
+
end
|
|
437
|
+
"You"
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
# A multipart/form-data body. fields: [name, value] or [name, io, {filename:, content_type:}].
|
|
441
|
+
def multipart(fields, boundary)
|
|
442
|
+
body = +"".b
|
|
443
|
+
fields.each do |name, value, file|
|
|
444
|
+
body << "--#{boundary}\r\n".b
|
|
445
|
+
if file
|
|
446
|
+
body << %(Content-Disposition: form-data; name="#{name}"; filename="#{file[:filename]}"\r\n).b
|
|
447
|
+
body << "Content-Type: #{file[:content_type]}\r\n\r\n".b
|
|
448
|
+
body << value.read.to_s.b
|
|
449
|
+
else
|
|
450
|
+
body << %(Content-Disposition: form-data; name="#{name}"\r\n\r\n).b
|
|
451
|
+
body << value.to_s.b
|
|
452
|
+
end
|
|
453
|
+
body << "\r\n".b
|
|
454
|
+
end
|
|
455
|
+
body << "--#{boundary}--\r\n".b
|
|
456
|
+
body
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
def rails_env
|
|
460
|
+
return nil unless defined?(::Rails) && ::Rails.respond_to?(:env)
|
|
461
|
+
|
|
462
|
+
::Rails.env.to_s
|
|
463
|
+
end
|
|
464
|
+
|
|
465
|
+
def log_warn(message)
|
|
466
|
+
logger = defined?(::Rails) && ::Rails.respond_to?(:logger) ? ::Rails.logger : nil
|
|
467
|
+
logger&.warn("[studio][voice] #{message}")
|
|
468
|
+
end
|
|
469
|
+
end
|
|
470
|
+
end
|
data/lib/studio.rb
CHANGED
|
@@ -47,6 +47,8 @@ require "studio/auth_labels"
|
|
|
47
47
|
require "studio/profile_sections"
|
|
48
48
|
require "studio/profile_image"
|
|
49
49
|
require "studio/voice_match"
|
|
50
|
+
require "studio/voice_capture"
|
|
51
|
+
require "studio/voice_transcription"
|
|
50
52
|
require "studio/oauth_identity"
|
|
51
53
|
require "studio/google_sign_in"
|
|
52
54
|
require "studio/newsletter"
|
|
@@ -576,6 +578,42 @@ module Studio
|
|
|
576
578
|
# refuses a voice saved by any other embedder, so a page cannot pick its own.
|
|
577
579
|
mattr_accessor :voice_embedder, default: "wespeaker"
|
|
578
580
|
|
|
581
|
+
# Understand (docs/VOICE_TRANSCRIPTION.md): POST /voice/takes turns a signed-in
|
|
582
|
+
# user's take into {text, speaker, score}. OFF BY DEFAULT: every take costs a
|
|
583
|
+
# speech-to-text call, so an app turns it on when a page posts takes.
|
|
584
|
+
mattr_accessor :voice_transcription, default: false
|
|
585
|
+
|
|
586
|
+
# Which transcriber answers: "openai" or "mock". nil means
|
|
587
|
+
# STUDIO_VOICE_TRANSCRIBER, else "openai" in production and "mock" elsewhere
|
|
588
|
+
# (Studio::VoiceTranscription.adapter_name). Tests never reach the vendor:
|
|
589
|
+
# VOICE_NO_LIVE_CALLS is armed in every test environment.
|
|
590
|
+
mattr_accessor :voice_transcriber, default: nil
|
|
591
|
+
|
|
592
|
+
# The OpenAI key for the real transcriber, voice's OWN key. nil reads
|
|
593
|
+
# STUDIO_VOICE_OPENAI_API_KEY, and never the app's shared OPENAI_API_KEY (on
|
|
594
|
+
# the hub that is the image key, with its own budget). Missing, the endpoint
|
|
595
|
+
# answers 503 with a line the page can show; nothing crashes.
|
|
596
|
+
mattr_accessor :voice_openai_api_key, default: nil
|
|
597
|
+
|
|
598
|
+
# One take's whole budget at the vendor, in seconds: the first model AND the
|
|
599
|
+
# fallback share it, so a take never holds a web thread longer than this
|
|
600
|
+
# (Studio::VoiceTranscription::OpenAI). A timeout or a vendor 429 is not
|
|
601
|
+
# retried.
|
|
602
|
+
mattr_accessor :voice_transcribe_deadline, default: 25
|
|
603
|
+
|
|
604
|
+
# POST /voice/takes per signed-in user: at most `to` takes in any `within`
|
|
605
|
+
# seconds (a fixed window), else a JSON 429 with retry_after. Every take is a
|
|
606
|
+
# paid vendor call. `store:` (optional) is the counter's cache; nil uses
|
|
607
|
+
# Rails.cache, or a per-process memory store when Rails.cache is a NullStore.
|
|
608
|
+
# false turns the limit off.
|
|
609
|
+
mattr_accessor :voice_takes_rate_limit, default: { to: 20, within: 600 }
|
|
610
|
+
|
|
611
|
+
# The speech-to-text models (Voicebox's measured pair) and the language both
|
|
612
|
+
# are told to hear. A blank fallback turns the one retry off.
|
|
613
|
+
mattr_accessor :voice_transcribe_model, default: "gpt-4o-transcribe"
|
|
614
|
+
mattr_accessor :voice_transcribe_fallback_model, default: "whisper-1"
|
|
615
|
+
mattr_accessor :voice_transcribe_language, default: "en"
|
|
616
|
+
|
|
579
617
|
# How long a freshly minted magic link stays live.
|
|
580
618
|
mattr_accessor :magic_link_ttl, default: 15.minutes
|
|
581
619
|
|
|
@@ -1955,6 +1993,13 @@ module Studio
|
|
|
1955
1993
|
delete "profile/voice", to: "studio/profiles#delete_voice"
|
|
1956
1994
|
end
|
|
1957
1995
|
|
|
1996
|
+
# Understand: one signed-in take in, {text, speaker, score} out
|
|
1997
|
+
# (Studio::VoiceTakesController, docs/VOICE_TRANSCRIPTION.md). Drawn only
|
|
1998
|
+
# for an app that turns it on, because every take is a paid vendor call.
|
|
1999
|
+
if Studio.voice_transcription
|
|
2000
|
+
post "voice/takes", to: "studio/voice_takes#create", as: :studio_voice_takes
|
|
2001
|
+
end
|
|
2002
|
+
|
|
1958
2003
|
resources :error_logs, only: [:index, :show]
|
|
1959
2004
|
|
|
1960
2005
|
# Admin
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: studio-engine
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.122.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Alex McRitchie
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-11-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rails
|
|
@@ -473,6 +473,7 @@ files:
|
|
|
473
473
|
- app/controllers/studio/session_states_controller.rb
|
|
474
474
|
- app/controllers/studio/site_identities_controller.rb
|
|
475
475
|
- app/controllers/studio/surveys_controller.rb
|
|
476
|
+
- app/controllers/studio/voice_takes_controller.rb
|
|
476
477
|
- app/controllers/style_controller.rb
|
|
477
478
|
- app/controllers/theme_settings_controller.rb
|
|
478
479
|
- app/helpers/studio/admin_models_table_helper.rb
|
|
@@ -507,12 +508,14 @@ files:
|
|
|
507
508
|
- app/javascript/studio/controllers/knowledge_transcript_controller.js
|
|
508
509
|
- app/javascript/studio/controllers/link_preview_card_controller.js
|
|
509
510
|
- app/javascript/studio/controllers/link_sidebar_controller.js
|
|
511
|
+
- app/javascript/studio/controllers/microphone_chip_controller.js
|
|
510
512
|
- app/javascript/studio/controllers/modal_host_controller.js
|
|
511
513
|
- app/javascript/studio/controllers/nav_collapse_controller.js
|
|
512
514
|
- app/javascript/studio/controllers/style_modals_controller.js
|
|
513
515
|
- app/javascript/studio/controllers/survey_controller.js
|
|
514
516
|
- app/javascript/studio/controllers/toast_controller.js
|
|
515
517
|
- app/javascript/studio/controllers/voice_profile_controller.js
|
|
518
|
+
- app/javascript/studio/controllers/voice_recorder_controller.js
|
|
516
519
|
- app/javascript/studio/cropper.js
|
|
517
520
|
- app/javascript/studio/email_banner.js
|
|
518
521
|
- app/javascript/studio/email_banner_scale.js
|
|
@@ -526,11 +529,14 @@ files:
|
|
|
526
529
|
- app/javascript/studio/image_upload.js
|
|
527
530
|
- app/javascript/studio/knowledge_transcript.js
|
|
528
531
|
- app/javascript/studio/lazy_controllers.js
|
|
532
|
+
- app/javascript/studio/level_meter.js
|
|
529
533
|
- app/javascript/studio/leveling_activity.js
|
|
530
534
|
- app/javascript/studio/link_preview_card.js
|
|
531
535
|
- app/javascript/studio/link_sidebar.js
|
|
532
536
|
- app/javascript/studio/loader.js
|
|
533
537
|
- app/javascript/studio/local_path.js
|
|
538
|
+
- app/javascript/studio/microphone.js
|
|
539
|
+
- app/javascript/studio/microphone_chip.js
|
|
534
540
|
- app/javascript/studio/modal_countdown.js
|
|
535
541
|
- app/javascript/studio/modal_host.js
|
|
536
542
|
- app/javascript/studio/nav_collapse.js
|
|
@@ -547,6 +553,7 @@ files:
|
|
|
547
553
|
- app/javascript/studio/toast.js
|
|
548
554
|
- app/javascript/studio/vendor/stimulus.js
|
|
549
555
|
- app/javascript/studio/voice_profile.js
|
|
556
|
+
- app/javascript/studio/voice_recorder.js
|
|
550
557
|
- app/jobs/error_log_cleanup_job.rb
|
|
551
558
|
- app/jobs/studio/email_delivery_job.rb
|
|
552
559
|
- app/jobs/studio/health_report_job.rb
|
|
@@ -759,6 +766,10 @@ files:
|
|
|
759
766
|
- app/views/studio/surveys/show.html.erb
|
|
760
767
|
- app/views/studio/surveys/sign_in_required.html.erb
|
|
761
768
|
- app/views/studio/surveys/thanks.html.erb
|
|
769
|
+
- app/views/studio/voice/_microphone_chip.html.erb
|
|
770
|
+
- app/views/studio/voice/_microphone_help.html.erb
|
|
771
|
+
- app/views/studio/voice/_recorder.html.erb
|
|
772
|
+
- app/views/studio/voice/_styles.html.erb
|
|
762
773
|
- app/views/style/_components.html.erb
|
|
763
774
|
- app/views/style/_host.html.erb
|
|
764
775
|
- app/views/style/_modal_specimen.html.erb
|
|
@@ -893,7 +904,9 @@ files:
|
|
|
893
904
|
- lib/studio/ui_primitives.rb
|
|
894
905
|
- lib/studio/username_generator.rb
|
|
895
906
|
- lib/studio/version.rb
|
|
907
|
+
- lib/studio/voice_capture.rb
|
|
896
908
|
- lib/studio/voice_match.rb
|
|
909
|
+
- lib/studio/voice_transcription.rb
|
|
897
910
|
- lib/tasks/studio_email.rake
|
|
898
911
|
- lib/tasks/studio_knowledge.rake
|
|
899
912
|
- lib/tasks/studio_node.rake
|