studio-engine 0.119.1 → 0.121.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +61 -0
- data/README.md +3 -0
- data/app/controllers/studio/profiles_controller.rb +56 -0
- data/app/controllers/studio/voice_takes_controller.rb +93 -0
- data/app/javascript/studio/controllers/microphone_chip_controller.js +21 -0
- data/app/javascript/studio/controllers/voice_profile_controller.js +20 -0
- data/app/javascript/studio/controllers/voice_recorder_controller.js +29 -0
- data/app/javascript/studio/level_meter.js +167 -0
- data/app/javascript/studio/microphone.js +250 -0
- data/app/javascript/studio/microphone_chip.js +370 -0
- data/app/javascript/studio/stimulus.js +10 -2
- data/app/javascript/studio/voice_profile.js +444 -0
- data/app/javascript/studio/voice_recorder.js +352 -0
- data/app/views/studio/profiles/_voice_section.html.erb +108 -0
- data/app/views/studio/voice/_microphone_chip.html.erb +45 -0
- data/app/views/studio/voice/_microphone_help.html.erb +63 -0
- data/app/views/studio/voice/_recorder.html.erb +34 -0
- data/app/views/studio/voice/_styles.html.erb +90 -0
- data/db/migrate/20261029120000_add_voice_profile_to_users.rb +44 -0
- data/lib/studio/engine.rb +10 -0
- data/lib/studio/profile_sections.rb +11 -0
- data/lib/studio/version.rb +1 -1
- data/lib/studio/voice_capture.rb +95 -0
- data/lib/studio/voice_match.rb +238 -0
- data/lib/studio/voice_transcription.rb +404 -0
- data/lib/studio.rb +57 -0
- metadata +20 -2
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
<%# The voice capture layer's scoped CSS (docs/VOICE_CAPTURE.md), rendered by
|
|
2
|
+
the chip and the recorder partials. Scoped classes rather than utilities, as
|
|
3
|
+
in studio/profiles/_voice_section: the chip, the dot, the ring and the help
|
|
4
|
+
drawing must render in an app whose Tailwind build never saw these files.
|
|
5
|
+
Every colour is a theme token (the primary through --color-primary-500-rgb
|
|
6
|
+
and the CTA fill under its label), so dark mode and each app's palette come
|
|
7
|
+
for free. Tap targets are 44px. A page rendering both partials carries this
|
|
8
|
+
block twice, which is harmless. %>
|
|
9
|
+
<style>
|
|
10
|
+
.studio-mic { display: flex; width: 100%; flex-direction: column; align-items: center; gap: .5rem; }
|
|
11
|
+
.studio-mic-chip { display: inline-flex; min-height: 44px; max-width: 100%; align-items: center; gap: .5rem;
|
|
12
|
+
border-radius: 1.5rem; border: 1px solid var(--color-border, rgb(127 127 127 / .25));
|
|
13
|
+
background: var(--color-inset, rgb(127 127 127 / .12)); padding: .5rem 1rem; font-size: .875rem; font-weight: 600;
|
|
14
|
+
color: var(--color-text, inherit); cursor: pointer; text-align: left; }
|
|
15
|
+
.studio-mic-chip:focus-visible { outline: 3px solid rgb(var(--color-primary-500-rgb, 99 102 241) / .5); outline-offset: 2px; }
|
|
16
|
+
.studio-mic-icon { width: 1rem; height: 1rem; flex-shrink: 0; color: var(--color-text-secondary, currentColor); }
|
|
17
|
+
.studio-mic-name { min-width: 0; overflow-wrap: anywhere; }
|
|
18
|
+
.studio-mic-caret { width: .75rem; height: .75rem; flex-shrink: 0; color: var(--color-text-muted, currentColor); }
|
|
19
|
+
.studio-mic-chip[aria-expanded="true"] .studio-mic-caret { transform: rotate(180deg); }
|
|
20
|
+
[data-microphone-state="denied"] .studio-mic-chip { border-color: rgb(var(--color-primary-500-rgb, 99 102 241)); }
|
|
21
|
+
/* The live dot: hollow while nothing records; while a take records it fills
|
|
22
|
+
with the primary colour as loud as the voice (--studio-mic-level, 0..1). */
|
|
23
|
+
.studio-mic-dot { width: .625rem; height: .625rem; flex-shrink: 0; border-radius: 9999px; border: 2px solid currentColor; opacity: .35; }
|
|
24
|
+
[data-microphone-live="true"] .studio-mic-dot { opacity: 1; border-color: rgb(var(--color-primary-500-rgb, 99 102 241));
|
|
25
|
+
background-color: rgb(var(--color-primary-500-rgb, 99 102 241) / calc(.15 + var(--studio-mic-level, 0) * .85));
|
|
26
|
+
transform: scale(calc(1 + var(--studio-mic-level, 0) * .5)); transition: transform 80ms linear, background-color 80ms linear; }
|
|
27
|
+
.studio-mic-panel { width: 100%; max-width: 24rem; padding: .75rem; text-align: left; display: grid; gap: .5rem; }
|
|
28
|
+
.studio-mic-panel[hidden] { display: none; }
|
|
29
|
+
.studio-mic-note { font-size: .875rem; color: var(--color-text-muted, inherit); }
|
|
30
|
+
.studio-mic-list { list-style: none; margin: 0; padding: 0; display: grid; gap: .25rem; }
|
|
31
|
+
.studio-mic-option { display: flex; min-height: 44px; width: 100%; align-items: center; gap: .5rem; border-radius: .75rem;
|
|
32
|
+
border: 1px solid transparent; background: transparent; padding: .5rem .75rem; text-align: left; font-size: .875rem;
|
|
33
|
+
color: var(--color-text, inherit); cursor: pointer; }
|
|
34
|
+
.studio-mic-option:hover { background-color: rgb(var(--color-primary-500-rgb, 99 102 241) / .08); }
|
|
35
|
+
.studio-mic-option[aria-current="true"] { border-color: rgb(var(--color-primary-500-rgb, 99 102 241));
|
|
36
|
+
background-color: rgb(var(--color-primary-500-rgb, 99 102 241) / .12); font-weight: 600; }
|
|
37
|
+
.studio-mic-check { width: 1rem; flex-shrink: 0; font-weight: 700; color: rgb(var(--color-primary-500-rgb, 99 102 241)); }
|
|
38
|
+
.studio-mic-option-label { min-width: 0; flex: 1; overflow-wrap: anywhere; }
|
|
39
|
+
.studio-mic-in-use { flex-shrink: 0; border-radius: 9999px; padding: .125rem .5rem; font-size: .75rem; font-weight: 700;
|
|
40
|
+
background: var(--color-cta-fill, var(--color-cta, rgb(var(--color-primary-500-rgb, 99 102 241)))); color: var(--btn-primary-fg, #fff); }
|
|
41
|
+
.studio-mic-help-link { min-height: 44px; width: 100%; border-radius: .75rem; padding: 0 .75rem; text-align: left; font-size: .875rem;
|
|
42
|
+
font-weight: 600; color: var(--color-text-secondary, inherit); text-decoration: underline; background: transparent; border: 0; cursor: pointer; }
|
|
43
|
+
/* The help dialog: a native modal <dialog>, so the page behind it is inert. */
|
|
44
|
+
.studio-mic-help { border: 0; padding: 0; margin: auto; width: min(28rem, calc(100vw - 2rem)); max-height: calc(100dvh - 2rem);
|
|
45
|
+
overflow-y: auto; text-align: left; color: var(--color-text-body, inherit); background: var(--color-surface, Canvas); border-radius: 1rem; }
|
|
46
|
+
.studio-mic-help::backdrop { background-color: rgb(0 0 0 / .55); }
|
|
47
|
+
.studio-mic-help-body { display: grid; gap: 1rem; padding: 1.25rem; }
|
|
48
|
+
.studio-mic-help-lead { display: grid; gap: .25rem; border-radius: .75rem; background: var(--color-inset, rgb(127 127 127 / .12)); padding: .75rem; }
|
|
49
|
+
.studio-mic-help-lead[hidden], .studio-mic-help-chrome[hidden], .studio-mic-help-other[hidden] { display: none; }
|
|
50
|
+
.studio-mic-help-headline { font-weight: 600; color: var(--color-text, inherit); margin: 0; }
|
|
51
|
+
.studio-mic-help-tip { font-size: .875rem; color: var(--color-text-secondary, inherit); margin: 0; }
|
|
52
|
+
.studio-mic-help-title { font-size: 1.25rem; font-weight: 700; color: var(--color-text, inherit); margin: 0; }
|
|
53
|
+
.studio-mic-help-steps { list-style: none; margin: 0; padding: 0; display: grid; gap: .5rem; }
|
|
54
|
+
.studio-mic-help-steps li { display: flex; align-items: flex-start; gap: .75rem; }
|
|
55
|
+
.studio-mic-step { display: flex; width: 1.5rem; height: 1.5rem; flex-shrink: 0; align-items: center; justify-content: center;
|
|
56
|
+
border-radius: 9999px; font-size: .75rem; font-weight: 700;
|
|
57
|
+
background: var(--color-cta-fill, var(--color-cta, rgb(var(--color-primary-500-rgb, 99 102 241)))); color: var(--btn-primary-fg, #fff); }
|
|
58
|
+
.studio-mic-help-actions { display: flex; justify-content: flex-end; margin: 0; }
|
|
59
|
+
.studio-mic-help-close { min-height: 44px; padding: 0 1.5rem; }
|
|
60
|
+
/* The drawn address bar: never a screenshot, every stroke a token. */
|
|
61
|
+
.studio-mic-mock { width: 100%; height: auto; color: var(--color-text, currentColor); }
|
|
62
|
+
.studio-mic-mock .mock-bar, .studio-mic-mock .mock-panel { fill: var(--color-inset, rgb(127 127 127 / .12)); stroke: currentColor; stroke-opacity: .25; }
|
|
63
|
+
.studio-mic-mock .mock-ring { fill: none; stroke: rgb(var(--color-primary-500-rgb, 99 102 241)); stroke-width: 3; }
|
|
64
|
+
.studio-mic-mock .mock-ink { stroke: currentColor; }
|
|
65
|
+
.studio-mic-mock .mock-knob { fill: currentColor; stroke: none; }
|
|
66
|
+
.studio-mic-mock .mock-text { fill: currentColor; font-family: inherit; }
|
|
67
|
+
.studio-mic-mock .mock-on, .studio-mic-mock .mock-badge { fill: rgb(var(--color-primary-500-rgb, 99 102 241)); }
|
|
68
|
+
.studio-mic-mock .mock-knob-on, .studio-mic-mock .mock-badge-text { fill: var(--color-surface, Canvas); }
|
|
69
|
+
|
|
70
|
+
/* The record button: a ring as loud as the voice (--studio-recorder-level). */
|
|
71
|
+
.studio-recorder { display: flex; flex-direction: column; align-items: center; gap: .75rem; text-align: center; }
|
|
72
|
+
.studio-recorder-button { min-height: 44px; min-width: 12rem; max-width: 100%; border: 0; border-radius: 9999px; padding: .875rem 1.75rem;
|
|
73
|
+
font-weight: 700; cursor: pointer; touch-action: none; -webkit-user-select: none; user-select: none;
|
|
74
|
+
color: var(--btn-primary-fg, #fff); background: var(--color-cta-fill, var(--color-cta, rgb(var(--color-primary-500-rgb, 99 102 241))));
|
|
75
|
+
box-shadow: 0 0 0 calc(var(--studio-recorder-level, 0) * 14px) rgb(var(--color-primary-500-rgb, 99 102 241) / .3);
|
|
76
|
+
transition: box-shadow 80ms linear; }
|
|
77
|
+
.studio-recorder-button:focus-visible { outline: 3px solid rgb(var(--color-primary-500-rgb, 99 102 241) / .5); outline-offset: 3px; }
|
|
78
|
+
.studio-recorder-button:disabled { opacity: .5; cursor: not-allowed; }
|
|
79
|
+
[data-voice-recorder-state="recording"] .studio-recorder-button { background: var(--color-danger-fill, var(--color-danger, #dc2626)); }
|
|
80
|
+
.studio-recorder-timer { font-variant-numeric: tabular-nums; font-size: .875rem; color: var(--color-text-muted, inherit); }
|
|
81
|
+
.studio-recorder-status { min-height: 2.5rem; max-width: 24rem; margin: 0; color: var(--color-text-body, inherit); }
|
|
82
|
+
.studio-recorder-failed { display: none; font-size: .8125rem; color: var(--color-text-secondary, inherit); }
|
|
83
|
+
[data-studio-controller-failed~="voice-recorder"] .studio-recorder-failed { display: block; }
|
|
84
|
+
.studio-mic-failed { display: none; margin: 0; font-size: .8125rem; color: var(--color-text-secondary, inherit); }
|
|
85
|
+
[data-studio-controller-failed~="microphone-chip"] .studio-mic-failed { display: block; }
|
|
86
|
+
@media (prefers-reduced-motion: reduce) {
|
|
87
|
+
[data-microphone-live="true"] .studio-mic-dot { transform: none; transition: none; }
|
|
88
|
+
.studio-recorder-button { transition: none; }
|
|
89
|
+
}
|
|
90
|
+
</style>
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# The voice row on /profile (Studio.voice_profile): one speaker embedding on the
|
|
2
|
+
# user's own record, the model it came from, and when it was saved.
|
|
3
|
+
#
|
|
4
|
+
# A HOST-TABLE MIGRATION, the engine's second after
|
|
5
|
+
# add_standard_user_profile_columns, and built the same way for the same reason:
|
|
6
|
+
# one definition the apps install (`bin/rails studio_engine:install:migrations`)
|
|
7
|
+
# instead of a hand-written copy per app.
|
|
8
|
+
#
|
|
9
|
+
# voice_embedding jsonb the averaged unit vector (256 floats for
|
|
10
|
+
# the wespeaker embedder). A pattern of the
|
|
11
|
+
# voice, never audio: the takes stay on the
|
|
12
|
+
# device.
|
|
13
|
+
# voice_embedding_model string which embedder and revision made it
|
|
14
|
+
# (Studio::VoiceMatch::EMBEDDERS ids). Vectors
|
|
15
|
+
# from different models are not comparable,
|
|
16
|
+
# so the matcher filters on this.
|
|
17
|
+
# voice_embedding_updated_at datetime
|
|
18
|
+
#
|
|
19
|
+
# Every add is `if_not_exists`, and an app with no users table is skipped, as in
|
|
20
|
+
# the standard-columns migration.
|
|
21
|
+
#
|
|
22
|
+
# THE DOWN DROPS THEM, unlike the standard-columns migration, which refuses. That
|
|
23
|
+
# one could not tell its columns from columns the host already owned
|
|
24
|
+
# (first_name, birth_year). These three names are the engine's own coinage, and
|
|
25
|
+
# no consumer had any of them when this shipped (mcritchie-studio,
|
|
26
|
+
# turf-monster, mcritchie-industries, cyvasse and moms-app checked 2026-10-29),
|
|
27
|
+
# so a drop removes only what this migration added.
|
|
28
|
+
class AddVoiceProfileToUsers < ActiveRecord::Migration[7.2]
|
|
29
|
+
def up
|
|
30
|
+
return unless table_exists?(:users)
|
|
31
|
+
|
|
32
|
+
add_column :users, :voice_embedding, :jsonb, if_not_exists: true
|
|
33
|
+
add_column :users, :voice_embedding_model, :string, if_not_exists: true
|
|
34
|
+
add_column :users, :voice_embedding_updated_at, :datetime, if_not_exists: true
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def down
|
|
38
|
+
return unless table_exists?(:users)
|
|
39
|
+
|
|
40
|
+
remove_column :users, :voice_embedding_updated_at, if_exists: true
|
|
41
|
+
remove_column :users, :voice_embedding_model, if_exists: true
|
|
42
|
+
remove_column :users, :voice_embedding, if_exists: true
|
|
43
|
+
end
|
|
44
|
+
end
|
data/lib/studio/engine.rb
CHANGED
|
@@ -299,6 +299,16 @@ module Studio
|
|
|
299
299
|
app.config.middleware.use Studio::GoogleSignIn::Middleware
|
|
300
300
|
end
|
|
301
301
|
|
|
302
|
+
# The voice row's vectors stay out of every consumer's request log and
|
|
303
|
+
# User#inspect: POST /profile/voice carries voice[embeddings], and the user
|
|
304
|
+
# holds voice_embedding (Voicebox logged vectors until it filtered them).
|
|
305
|
+
# `(\A|\.)` also masks the JSON param wrapper's copy (profile.voice.embeddings).
|
|
306
|
+
# POST /voice/takes carries voice[embedding] (one take's vector) and
|
|
307
|
+
# voice[audio] (the take itself); both are masked the same way.
|
|
308
|
+
initializer "studio.voice_profile.filter_parameters" do |app|
|
|
309
|
+
app.config.filter_parameters += [/(\A|\.)voice\.(embeddings|embedding|audio)\z/, /\Avoice_embedding\z/]
|
|
310
|
+
end
|
|
311
|
+
|
|
302
312
|
initializer "studio.footer_only_eager_load" do
|
|
303
313
|
next if Studio.active_record?
|
|
304
314
|
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
# and the engine fails to boot. Declaring the dependency where it is used makes
|
|
7
7
|
# the pair immune to a future reorder of that require list.
|
|
8
8
|
require_relative "newsletter"
|
|
9
|
+
require_relative "voice_match"
|
|
9
10
|
|
|
10
11
|
# Resolves Studio.profile_sections — the rows that make up the shared /profile
|
|
11
12
|
# page — for a given view context. Pure Ruby (no Rails dependency) so the unit
|
|
@@ -100,6 +101,16 @@ module Studio
|
|
|
100
101
|
requires: Studio::Newsletter::COLUMNS,
|
|
101
102
|
modals: "studio/profiles/newsletter_modals" },
|
|
102
103
|
|
|
104
|
+
# The voice: three phrases read once, so an agent can tell who is
|
|
105
|
+
# talking. OPT-IN (Studio.voice_profile), because it fetches a speaker
|
|
106
|
+
# model from Hugging Face and stores a voice pattern, and a host's CSP and
|
|
107
|
+
# privacy page must say so first. Gated on the columns too: the engine
|
|
108
|
+
# migration add_voice_profile_to_users installs them.
|
|
109
|
+
{ key: :voice, title: "Your voice", page: :show,
|
|
110
|
+
partial: "studio/profiles/voice_section",
|
|
111
|
+
requires: Studio::VoiceMatch::USER_COLUMNS,
|
|
112
|
+
if: -> { Studio.voice_profile } },
|
|
113
|
+
|
|
103
114
|
# --- the edit page ------------------------------------------------------
|
|
104
115
|
# These are FIELDS in one form with one Save, so they carry no buttons of
|
|
105
116
|
# their own. Email included: it needed a separate action only while it was
|
data/lib/studio/version.rb
CHANGED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# The words on the engine's voice capture layer: the microphone chip and its
|
|
4
|
+
# help dialog (studio/voice/_microphone_chip, _microphone_help), and the record
|
|
5
|
+
# button (studio/voice/_recorder). Ported from Voicebox's MicrophoneHelper and
|
|
6
|
+
# RecordingsHelper; the contract is docs/VOICE_CAPTURE.md.
|
|
7
|
+
#
|
|
8
|
+
# The chip names the microphone the page records from, so a switched-off
|
|
9
|
+
# headset Chrome picked on its own is plain to see, and lets the person pick
|
|
10
|
+
# another. A host passes `copy:` to either partial to change any line; its
|
|
11
|
+
# keys merge over these, one level deep per section.
|
|
12
|
+
module Studio
|
|
13
|
+
module VoiceCapture
|
|
14
|
+
# Where the chip remembers the pick (localStorage), unless a partial is given
|
|
15
|
+
# storage_key:. Mirrors STORAGE_KEY in app/javascript/studio/microphone.js.
|
|
16
|
+
STORAGE_KEY = "studio.microphone"
|
|
17
|
+
|
|
18
|
+
MICROPHONE_COPY = {
|
|
19
|
+
# Before the person allows the microphone the browser gives no names.
|
|
20
|
+
"prompt" => { "chip" => "Microphone: tap Record to allow",
|
|
21
|
+
"note" => "Tap Record once and allow the microphone. Then you can pick one here." },
|
|
22
|
+
# The microphone icon says what the name is, so the name gets the whole
|
|
23
|
+
# width at 390px; a screen reader hears "aria" in full.
|
|
24
|
+
"ready" => { "chip" => "%{microphone}", "aria" => "Microphone: %{microphone}. Tap to pick another.",
|
|
25
|
+
"note" => "Pick the microphone to record with. Your next take uses it." },
|
|
26
|
+
"denied" => { "chip" => "Microphone not allowed",
|
|
27
|
+
"note" => "This site isn't allowed to use the microphone. Tap the icon beside the web address, " \
|
|
28
|
+
"set Microphone to Allow, then reload this page." },
|
|
29
|
+
"unsupported" => { "chip" => "No microphone here",
|
|
30
|
+
"note" => "This browser can't list microphones. Try Safari or Chrome." },
|
|
31
|
+
"fell_back" => "That microphone isn't connected, so I'm using %{microphone}.",
|
|
32
|
+
"empty" => "No microphones found. Plug one in and it shows up here.",
|
|
33
|
+
"in_use" => "In use",
|
|
34
|
+
# Shown only when the chip's controller failed to load.
|
|
35
|
+
"unavailable" => "Picking a microphone didn't load. Reload to try again.",
|
|
36
|
+
# The help dialog. Switching happens on the page, by device id, so the
|
|
37
|
+
# dialog never sends anyone to Chrome to switch: Chrome's settings are
|
|
38
|
+
# where the microphone is ALLOWED, and where a person checks it when a
|
|
39
|
+
# take stays silent.
|
|
40
|
+
"help" => {
|
|
41
|
+
"link" => "Help with microphones",
|
|
42
|
+
"title_chrome" => "How to change your microphone in Chrome",
|
|
43
|
+
"title_other" => "How to change your microphone",
|
|
44
|
+
"steps_chrome" => [
|
|
45
|
+
"Click the icon left of the address.",
|
|
46
|
+
"Turn Microphone on.",
|
|
47
|
+
"Make sure your microphone is allowed.",
|
|
48
|
+
"Reload the page."
|
|
49
|
+
],
|
|
50
|
+
"other" => "Allow the microphone when your browser asks, or in its settings for this site, " \
|
|
51
|
+
"then pick your microphone here and reload.",
|
|
52
|
+
"unnamed" => "the microphone",
|
|
53
|
+
"switched" => "Switched to %{microphone} ✓",
|
|
54
|
+
"switched_tip_chrome" => "If Chrome asks or it stays silent, here's how to check Chrome's microphone.",
|
|
55
|
+
"switched_tip_other" => "If your browser asks or it stays silent, here's how to check its microphone.",
|
|
56
|
+
"silent" => "I couldn't hear anything from %{microphone}.",
|
|
57
|
+
"silent_tip" => "Pick another microphone in the list, or check that the browser may use it:",
|
|
58
|
+
"denied" => "This site isn't allowed to use the microphone yet.",
|
|
59
|
+
"denied_tip" => "Allow it, then reload:",
|
|
60
|
+
"close" => "Close"
|
|
61
|
+
}
|
|
62
|
+
}.freeze
|
|
63
|
+
|
|
64
|
+
# The record button's words per state. Mirrors COPY in
|
|
65
|
+
# app/javascript/studio/voice_recorder.js, which fills any state a page
|
|
66
|
+
# leaves out; the partial renders the idle words so the button reads right
|
|
67
|
+
# before the controller connects.
|
|
68
|
+
RECORDER_COPY = {
|
|
69
|
+
"idle" => { "button" => "Record", "status" => "" },
|
|
70
|
+
"recording" => { "button" => "Stop", "status" => "Listening..." },
|
|
71
|
+
"unavailable" => "Recording didn't load on this page. Reload to try again."
|
|
72
|
+
}.freeze
|
|
73
|
+
|
|
74
|
+
module_function
|
|
75
|
+
|
|
76
|
+
# The copy with a host's overrides merged in: a Hash section merges key by
|
|
77
|
+
# key, anything else replaces.
|
|
78
|
+
def microphone_copy(overrides = nil)
|
|
79
|
+
merge(MICROPHONE_COPY, overrides)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def recorder_copy(overrides = nil)
|
|
83
|
+
merge(RECORDER_COPY, overrides)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def merge(base, overrides)
|
|
87
|
+
return base if overrides.nil? || overrides.empty?
|
|
88
|
+
|
|
89
|
+
overrides.to_h.each_with_object(base.dup) do |(key, value), merged|
|
|
90
|
+
key = key.to_s
|
|
91
|
+
merged[key] = merged[key].is_a?(Hash) && value.is_a?(Hash) ? merged[key].merge(value.transform_keys(&:to_s)) : value
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
# Who is speaking: compares one take's speaker embedding with the voice
|
|
6
|
+
# embeddings saved on users' own records (the voice row on /profile, opt-in with
|
|
7
|
+
# Studio.voice_profile). Pure arithmetic on arrays of floats, with no Rails
|
|
8
|
+
# dependency, so the threshold logic is unit-tested without a browser, a model or
|
|
9
|
+
# a database.
|
|
10
|
+
#
|
|
11
|
+
# THE EMBEDDINGS COME FROM THE BROWSER. The profile page records three short
|
|
12
|
+
# phrases, turns each take into a vector on the person's own device
|
|
13
|
+
# (app/javascript/studio/voice_profile.js), drops the audio, and posts only the
|
|
14
|
+
# vectors. The server averages them here and stores one vector on the user. No
|
|
15
|
+
# audio is ever uploaded for this.
|
|
16
|
+
#
|
|
17
|
+
# Ported from Voicebox's SpeakerMatch (McRitchie-Studio/voicebox#10), where the
|
|
18
|
+
# embedder, the threshold and the averaging were measured.
|
|
19
|
+
module Studio
|
|
20
|
+
module VoiceMatch
|
|
21
|
+
# The speaker embedders the browser may run, by name. `id` is what is stored
|
|
22
|
+
# on the user beside the vector (users.voice_embedding_model): vectors from
|
|
23
|
+
# two models, or from two revisions of one model, are not comparable, so the
|
|
24
|
+
# matcher only ever compares a take with users whose id is the same.
|
|
25
|
+
#
|
|
26
|
+
# "wespeaker" is WeSpeaker's ResNet34 speaker model (trained on VoxCeleb),
|
|
27
|
+
# int8-quantized, run by transformers.js on onnxruntime-web. The model is
|
|
28
|
+
# about 6.7 MB and the runtime about 4.3 MB; the browser caches both after the
|
|
29
|
+
# first load. Its revision is pinned so THRESHOLD always describes the vectors
|
|
30
|
+
# it was tuned on. Changing the revision is changing the id, and every saved
|
|
31
|
+
# voice stops matching until its owner re-records.
|
|
32
|
+
#
|
|
33
|
+
# "fake" is for tests only: a deterministic vector seeded by a voice name the
|
|
34
|
+
# test sets on the page (window.studioFakeVoice), because Chrome's fake
|
|
35
|
+
# microphone cannot play chosen audio.
|
|
36
|
+
TRANSFORMERS_VERSION = "3.7.6"
|
|
37
|
+
|
|
38
|
+
# Where the browser imports transformers.js from, by full URL. NOT an
|
|
39
|
+
# importmap pin: every pin a host draws must be a fingerprinted local asset
|
|
40
|
+
# (the hub's AssetUrlsTest), and a ~1 MB runtime that fetches its own
|
|
41
|
+
# onnxruntime WebAssembly from the same jsDelivr path anyway gains nothing
|
|
42
|
+
# from being vendored. A host's CSP names this package path.
|
|
43
|
+
TRANSFORMERS_URL =
|
|
44
|
+
"https://cdn.jsdelivr.net/npm/@huggingface/transformers@#{TRANSFORMERS_VERSION}/dist/transformers.min.js"
|
|
45
|
+
|
|
46
|
+
EMBEDDERS = {
|
|
47
|
+
"wespeaker" => {
|
|
48
|
+
"id" => "wespeaker-voxceleb-resnet34-LM@6a61a18",
|
|
49
|
+
"size" => 256,
|
|
50
|
+
"model" => "onnx-community/wespeaker-voxceleb-resnet34-LM",
|
|
51
|
+
"revision" => "6a61a1833ff2583aabeba044f5c8221f00b67ceb",
|
|
52
|
+
"dtype" => "q8",
|
|
53
|
+
"runtime" => TRANSFORMERS_URL
|
|
54
|
+
}.freeze,
|
|
55
|
+
"fake" => {
|
|
56
|
+
"id" => "fake-256",
|
|
57
|
+
"size" => 256
|
|
58
|
+
}.freeze
|
|
59
|
+
}.freeze
|
|
60
|
+
|
|
61
|
+
# The columns the voice lives in on the host's users table, installed by the
|
|
62
|
+
# engine migration add_voice_profile_to_users.
|
|
63
|
+
USER_COLUMNS = %i[voice_embedding voice_embedding_model voice_embedding_updated_at].freeze
|
|
64
|
+
|
|
65
|
+
# What the setup asks the person to read: three short, phonetically varied
|
|
66
|
+
# sentences of about four seconds each. TAKES is their count, and the server
|
|
67
|
+
# refuses a save with any other number of vectors.
|
|
68
|
+
PHRASES = [
|
|
69
|
+
"The quick brown fox jumps over the lazy dog.",
|
|
70
|
+
"I'd like a cup of coffee and a slice of toast, please.",
|
|
71
|
+
"My voice is my own, and today is a good day to talk."
|
|
72
|
+
].freeze
|
|
73
|
+
|
|
74
|
+
TAKES = PHRASES.size
|
|
75
|
+
|
|
76
|
+
# The cosine similarity a take must reach to be named. Measured in Voicebox
|
|
77
|
+
# (2026-10-29) on the wespeaker embedder, enrolling five synthetic voices from
|
|
78
|
+
# three phrases each and probing with a fourth: same voice 0.85 - 0.91,
|
|
79
|
+
# different voice 0.04 - 0.53 (the closest pair two adult women). 0.6 sits
|
|
80
|
+
# above every different-voice score and well under every same-voice one.
|
|
81
|
+
THRESHOLD = 0.6
|
|
82
|
+
|
|
83
|
+
# user: the closest user at or above the threshold, else nil. score: that
|
|
84
|
+
# closest similarity, or nil when there was nothing to compare.
|
|
85
|
+
Result = Struct.new(:user, :score, keyword_init: true) do
|
|
86
|
+
def unknown? = user.nil?
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
NONE = Result.new(user: nil, score: nil).freeze
|
|
90
|
+
|
|
91
|
+
module_function
|
|
92
|
+
|
|
93
|
+
# The settings the page hands the browser for one embedder.
|
|
94
|
+
def browser_config(name)
|
|
95
|
+
config = EMBEDDERS.fetch(name.to_s)
|
|
96
|
+
{ "name" => name.to_s }.merge(config)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def embedder!(name)
|
|
100
|
+
EMBEDDERS.fetch(name.to_s) do
|
|
101
|
+
raise ArgumentError, "Studio.voice_embedder must be one of #{EMBEDDERS.keys.join(', ')} (got #{name.inspect})"
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def model_id(name) = embedder!(name).fetch("id")
|
|
106
|
+
|
|
107
|
+
def size(name) = embedder!(name).fetch("size")
|
|
108
|
+
|
|
109
|
+
# The closest user to `embedding` among `among`, or nil when nobody reaches
|
|
110
|
+
# the threshold. Only users whose saved vector came from `model` are
|
|
111
|
+
# compared. `among` is anything enumerable of users; an ActiveRecord scope is
|
|
112
|
+
# narrowed to that model in SQL first.
|
|
113
|
+
def identify(embedding, among:, model:, threshold: THRESHOLD)
|
|
114
|
+
match(embedding, among: among, model: model, threshold: threshold).user
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# identify, with the score: a caller that wants to say "someone new" needs to
|
|
118
|
+
# tell "nobody close" (a score below the threshold) from "nothing to compare".
|
|
119
|
+
def match(embedding, among:, model:, threshold: THRESHOLD)
|
|
120
|
+
probe = vector(embedding)
|
|
121
|
+
return NONE if probe.nil? || model.to_s.empty?
|
|
122
|
+
|
|
123
|
+
best = nil
|
|
124
|
+
best_score = nil
|
|
125
|
+
candidates(among, model).each do |user|
|
|
126
|
+
next unless user.respond_to?(:voice_embedding_model) && user.voice_embedding_model.to_s == model.to_s
|
|
127
|
+
|
|
128
|
+
saved = vector(user.voice_embedding)
|
|
129
|
+
next if saved.nil? || saved.size != probe.size
|
|
130
|
+
|
|
131
|
+
score = cosine(probe, saved)
|
|
132
|
+
next if score.nil?
|
|
133
|
+
|
|
134
|
+
if best_score.nil? || score > best_score
|
|
135
|
+
best = user
|
|
136
|
+
best_score = score
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
return NONE if best_score.nil?
|
|
140
|
+
|
|
141
|
+
Result.new(user: best_score >= threshold ? best : nil, score: best_score.round(4))
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Cosine similarity, -1..1; nil when the sizes differ or either vector has
|
|
145
|
+
# no length.
|
|
146
|
+
def cosine(a, b)
|
|
147
|
+
return nil unless a.size == b.size
|
|
148
|
+
|
|
149
|
+
dot = 0.0
|
|
150
|
+
norm_a = 0.0
|
|
151
|
+
norm_b = 0.0
|
|
152
|
+
a.each_with_index do |x, i|
|
|
153
|
+
y = b[i]
|
|
154
|
+
dot += x * y
|
|
155
|
+
norm_a += x * x
|
|
156
|
+
norm_b += y * y
|
|
157
|
+
end
|
|
158
|
+
return nil if norm_a.zero? || norm_b.zero?
|
|
159
|
+
|
|
160
|
+
dot / Math.sqrt(norm_a * norm_b)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# One profile vector from the setup takes: each take scaled to unit length,
|
|
164
|
+
# so a loud phrase does not outweigh a quiet one, then averaged and scaled to
|
|
165
|
+
# unit length again. nil when no take has any length (silence embeds to
|
|
166
|
+
# zeros), or the takes disagree on size.
|
|
167
|
+
def average(vectors)
|
|
168
|
+
units = Array(vectors).map { |v| normalize(v) }.compact
|
|
169
|
+
return nil if units.empty?
|
|
170
|
+
return nil unless units.all? { |u| u.size == units.first.size }
|
|
171
|
+
|
|
172
|
+
sum = Array.new(units.first.size, 0.0)
|
|
173
|
+
units.each { |u| u.each_with_index { |x, i| sum[i] += x } }
|
|
174
|
+
normalize(sum)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def normalize(vector)
|
|
178
|
+
return nil if vector.nil? || vector.empty?
|
|
179
|
+
|
|
180
|
+
norm = Math.sqrt(vector.sum { |x| x * x })
|
|
181
|
+
return nil if norm.zero? || !norm.finite?
|
|
182
|
+
|
|
183
|
+
vector.map { |x| x / norm }
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# A vector the browser sent, as floats, or nil when it is not `size` finite
|
|
187
|
+
# numbers. Accepts a JSON string or an already-parsed array.
|
|
188
|
+
def parse(raw, size:)
|
|
189
|
+
values = raw.is_a?(String) ? JSON.parse(raw) : raw
|
|
190
|
+
return nil unless values.is_a?(Array) && values.size == size
|
|
191
|
+
return nil unless values.all? { |x| x.is_a?(Numeric) && x.to_f.finite? }
|
|
192
|
+
|
|
193
|
+
values.map(&:to_f)
|
|
194
|
+
rescue JSON::ParserError
|
|
195
|
+
nil
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# The setup's takes, validated: exactly TAKES vectors of `size` finite
|
|
199
|
+
# numbers, each with some length. Returns the averaged unit vector, or nil.
|
|
200
|
+
def enrol(raw_takes, size:)
|
|
201
|
+
takes = raw_takes.is_a?(String) ? JSON.parse(raw_takes) : raw_takes
|
|
202
|
+
return nil unless takes.is_a?(Array) && takes.size == TAKES
|
|
203
|
+
|
|
204
|
+
vectors = takes.map { |take| parse(take, size: size) }
|
|
205
|
+
return nil if vectors.any?(&:nil?)
|
|
206
|
+
return nil if vectors.any? { |v| normalize(v).nil? }
|
|
207
|
+
|
|
208
|
+
average(vectors)
|
|
209
|
+
rescue JSON::ParserError
|
|
210
|
+
nil
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
# A stored embedding (jsonb array, or a JSON string on an adapter without
|
|
214
|
+
# json columns) as floats, or nil.
|
|
215
|
+
def vector(raw)
|
|
216
|
+
values = raw.is_a?(String) ? JSON.parse(raw) : raw
|
|
217
|
+
return nil unless values.is_a?(Array) && !values.empty?
|
|
218
|
+
return nil unless values.all? { |x| x.is_a?(Numeric) && x.to_f.finite? }
|
|
219
|
+
|
|
220
|
+
values.map(&:to_f)
|
|
221
|
+
rescue JSON::ParserError
|
|
222
|
+
nil
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def candidates(among, model)
|
|
226
|
+
return Array(among) unless among.respond_to?(:where)
|
|
227
|
+
|
|
228
|
+
# A scope or the model class itself. Narrowed in SQL only when the table
|
|
229
|
+
# has the column; the per-user check in #match holds either way.
|
|
230
|
+
klass = among.respond_to?(:klass) ? among.klass : among
|
|
231
|
+
if klass.respond_to?(:column_names) && klass.column_names.include?("voice_embedding_model")
|
|
232
|
+
among.where(voice_embedding_model: model.to_s)
|
|
233
|
+
else
|
|
234
|
+
among
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
end
|