omniai-google 3.19.0 → 3.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +107 -0
- data/lib/omniai/google/chat.rb +5 -0
- data/lib/omniai/google/client.rb +59 -5
- data/lib/omniai/google/config.rb +6 -0
- data/lib/omniai/google/embed.rb +6 -1
- data/lib/omniai/google/live/connection.rb +253 -0
- data/lib/omniai/google/live/event.rb +49 -0
- data/lib/omniai/google/live.rb +411 -0
- data/lib/omniai/google/transcribe/gemini.rb +218 -0
- data/lib/omniai/google/transcribe.rb +22 -0
- data/lib/omniai/google/transcribe_helpers.rb +4 -0
- data/lib/omniai/google/unknown_location_error.rb +17 -0
- data/lib/omniai/google/version.rb +1 -1
- data/lib/omniai/google/vertex.rb +25 -0
- metadata +21 -1
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OmniAI
|
|
4
|
+
module Google
|
|
5
|
+
# A Gemini Live API session on Vertex (BidiGenerateContent over a WebSocket): stream microphone audio in, receive
|
|
6
|
+
# speech, transcripts and tool calls back.
|
|
7
|
+
#
|
|
8
|
+
# Usage:
|
|
9
|
+
#
|
|
10
|
+
# session = client.live(model: "gemini-3.8-live", system: "...", tools: [RecordAnswer.new]) do |event|
|
|
11
|
+
# case event
|
|
12
|
+
# when OmniAI::Google::Live::Event::Audio then play(event.data)
|
|
13
|
+
# when OmniAI::Google::Live::Event::Closed then reconnect
|
|
14
|
+
# end
|
|
15
|
+
# end
|
|
16
|
+
# session.send_audio(pcm16_16khz)
|
|
17
|
+
# session.close
|
|
18
|
+
#
|
|
19
|
+
# Each session holds one socket and one reader thread. Events are yielded, and tools run, on that thread; the
|
|
20
|
+
# `send_*` methods may be called from any thread. The gem never logs or raises audio, transcripts or tool arguments.
|
|
21
|
+
class Live
|
|
22
|
+
# @!attribute code [Integer, nil] the WebSocket close code
|
|
23
|
+
# @!attribute reason [String, nil] Google's close reason; kept out of the message because it can echo content
|
|
24
|
+
class Error < OmniAI::Error
|
|
25
|
+
attr_reader :code, :reason
|
|
26
|
+
|
|
27
|
+
def initialize(message, code: nil, reason: nil)
|
|
28
|
+
super(message)
|
|
29
|
+
@code = code
|
|
30
|
+
@reason = reason
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
class ClosedError < Error; end
|
|
35
|
+
class TimeoutError < Error; end
|
|
36
|
+
|
|
37
|
+
module Model
|
|
38
|
+
GEMINI_3_8_LIVE = "gemini-3.8-live"
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
DEFAULT_MODEL = Model::GEMINI_3_8_LIVE
|
|
42
|
+
INPUT_MIME_TYPE = "audio/pcm;rate=16000"
|
|
43
|
+
PATH = "/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent"
|
|
44
|
+
SETUP_TIMEOUT = 15 # seconds
|
|
45
|
+
WRITE_TIMEOUT = 5 # seconds
|
|
46
|
+
BEHAVIORS = { blocking: "BLOCKING", non_blocking: "NON_BLOCKING" }.freeze
|
|
47
|
+
ACTIVITY = { "ACTIVITY_START" => :start, "ACTIVITY_END" => :end }.freeze
|
|
48
|
+
|
|
49
|
+
# Opens a session and waits for the server's setupComplete.
|
|
50
|
+
#
|
|
51
|
+
# @raise [Error] if the server rejects the setup or it times out
|
|
52
|
+
#
|
|
53
|
+
# @return [Live]
|
|
54
|
+
def self.open(...)
|
|
55
|
+
new(...).tap(&:open)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# @return [String, nil] the server's session id, once open
|
|
59
|
+
attr_reader :session_id
|
|
60
|
+
|
|
61
|
+
# @param client [OmniAI::Google::Client] must use Vertex credentials, a project and a location (e.g. "us")
|
|
62
|
+
# @param model [String]
|
|
63
|
+
# @param system [String, nil] the system instruction
|
|
64
|
+
# @param tools [Array<OmniAI::Tool>, nil] run automatically; each result is sent back as a tool response
|
|
65
|
+
# @param tool_behavior [Symbol, Hash] `:blocking` (default) or `:non_blocking`, or a Hash of tool name to either
|
|
66
|
+
# @param voice [String, nil] a prebuilt voice name, e.g. "Kore"
|
|
67
|
+
# @param transcription [Boolean] transcribe both the user's and the model's speech (default true)
|
|
68
|
+
# @param vad [Hash, nil] automatic activity detection: `disabled:`, `start_sensitivity:` / `end_sensitivity:`
|
|
69
|
+
# (`:high` or `:low`), `prefix_padding_ms:`, `silence_ms:`
|
|
70
|
+
# @param compression [Boolean, Hash] sliding-window context compression, which lifts the 15-minute session cap and
|
|
71
|
+
# bounds the context re-billed every turn. true uses Google's defaults (trigger at 80% of the context window);
|
|
72
|
+
# or pass `trigger_tokens:` / `target_tokens:`
|
|
73
|
+
# @param resumption [Boolean, String, nil] true to receive resumption handles, or a handle to resume
|
|
74
|
+
# @param write_timeout [Numeric] seconds a send may block before the session is closed
|
|
75
|
+
#
|
|
76
|
+
# @yieldparam event [Live::Event::*]
|
|
77
|
+
def initialize(client:, model: DEFAULT_MODEL, system: nil, tools: nil, tool_behavior: :blocking, voice: nil,
|
|
78
|
+
transcription: true, vad: nil, compression: false, resumption: nil, write_timeout: WRITE_TIMEOUT, &handler)
|
|
79
|
+
@client = client.for_model(model)
|
|
80
|
+
@model = model
|
|
81
|
+
@system = system
|
|
82
|
+
@tools = tools || []
|
|
83
|
+
@tool_behavior = tool_behavior
|
|
84
|
+
@voice = voice
|
|
85
|
+
@transcription = transcription
|
|
86
|
+
@vad = vad
|
|
87
|
+
@compression = compression
|
|
88
|
+
@resumption = resumption
|
|
89
|
+
@write_timeout = write_timeout
|
|
90
|
+
@handler = handler
|
|
91
|
+
@state = :new
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# @raise [Error]
|
|
95
|
+
#
|
|
96
|
+
# @return [Live]
|
|
97
|
+
def open
|
|
98
|
+
raise ArgumentError, "#{self.class.name} requires Vertex credentials" unless @client.credentials?
|
|
99
|
+
|
|
100
|
+
@ready = Queue.new
|
|
101
|
+
@connection = Connection.new(
|
|
102
|
+
url:,
|
|
103
|
+
headers: { "Authorization" => @client.send(:auth) }, # a fresh token per connection
|
|
104
|
+
write_timeout: @write_timeout,
|
|
105
|
+
on_open: method(:on_open),
|
|
106
|
+
on_message: method(:on_message),
|
|
107
|
+
on_close: method(:on_close)
|
|
108
|
+
)
|
|
109
|
+
log("LIVE connect #{host} #{@model}")
|
|
110
|
+
begin
|
|
111
|
+
@connection.connect
|
|
112
|
+
rescue Connection::ConnectTimeout => e
|
|
113
|
+
raise TimeoutError, "#{self.class.name} #{e.message} (model=#{@model})"
|
|
114
|
+
end
|
|
115
|
+
await_setup!
|
|
116
|
+
self
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# @return [Boolean]
|
|
120
|
+
def open?
|
|
121
|
+
@state == :open && @connection.open?
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# @param audio [String] raw PCM16 audio, 16 kHz, mono, little-endian
|
|
125
|
+
def send_audio(audio)
|
|
126
|
+
deliver(realtimeInput: { audio: { mimeType: INPUT_MIME_TYPE, data: Base64.strict_encode64(audio) } })
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# Tells the server the microphone paused, so it flushes any buffered audio.
|
|
130
|
+
def end_audio
|
|
131
|
+
deliver(realtimeInput: { audioStreamEnd: true })
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Sends a complete user turn of text, e.g. to re-seed context after reconnecting.
|
|
135
|
+
#
|
|
136
|
+
# @param text [String]
|
|
137
|
+
def send_text(text)
|
|
138
|
+
deliver(clientContent: { turns: [{ role: "user", parts: [{ text: }] }], turnComplete: true })
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def close
|
|
142
|
+
@connection&.close
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
private
|
|
146
|
+
|
|
147
|
+
# @param message [Hash]
|
|
148
|
+
#
|
|
149
|
+
# @raise [ClosedError, TimeoutError]
|
|
150
|
+
def deliver(message)
|
|
151
|
+
raise ClosedError, "#{self.class.name} session is closed (session=#{@session_id})" unless open?
|
|
152
|
+
|
|
153
|
+
@connection.send_text(JSON.generate(message))
|
|
154
|
+
rescue Connection::WriteTimeout
|
|
155
|
+
raise TimeoutError, "#{self.class.name} write timed out after #{@write_timeout}s (session=#{@session_id})"
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def await_setup!
|
|
159
|
+
result = @ready.pop(timeout: SETUP_TIMEOUT)
|
|
160
|
+
return if result == :ready
|
|
161
|
+
|
|
162
|
+
@connection.close
|
|
163
|
+
raise Error, "#{self.class.name} setup timed out (model=#{@model})" if result.nil?
|
|
164
|
+
|
|
165
|
+
code, reason, error = result
|
|
166
|
+
raise Error.new("#{self.class.name} setup failed (model=#{@model} code=#{code.inspect} " \
|
|
167
|
+
"error=#{error&.class.inspect})", code:, reason:)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def on_open
|
|
171
|
+
@connection.send_text(JSON.generate(setup))
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# @param data [String]
|
|
175
|
+
def on_message(data)
|
|
176
|
+
message, events = decode(data)
|
|
177
|
+
if (complete = message["setupComplete"])
|
|
178
|
+
@session_id = complete["sessionId"]
|
|
179
|
+
@state = :open
|
|
180
|
+
@ready << :ready
|
|
181
|
+
return
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
events.each do |event|
|
|
185
|
+
@handler&.call(event)
|
|
186
|
+
respond(event) if event.is_a?(Event::ToolCall)
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def on_close(code, reason, error)
|
|
191
|
+
state = @state
|
|
192
|
+
@state = :closed
|
|
193
|
+
log("LIVE close #{code || '-'} #{@session_id || '-'}")
|
|
194
|
+
if state == :open
|
|
195
|
+
@handler&.call(Event::Closed.new(code:, reason:, error:))
|
|
196
|
+
else
|
|
197
|
+
@ready << [code, reason, error]
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# Runs each called tool and sends the results. A raising tool answers with its error class only (never its
|
|
202
|
+
# message, which may carry arguments), and the session carries on.
|
|
203
|
+
#
|
|
204
|
+
# @param event [Event::ToolCall]
|
|
205
|
+
def respond(event)
|
|
206
|
+
responses = event.calls.map do |call|
|
|
207
|
+
tool = @tools.find { |candidate| candidate.name == call.name }
|
|
208
|
+
response =
|
|
209
|
+
if tool.nil?
|
|
210
|
+
{ error: "unknown tool: #{call.name}" }
|
|
211
|
+
else
|
|
212
|
+
begin
|
|
213
|
+
{ output: tool.call(call.arguments) }
|
|
214
|
+
rescue StandardError => e
|
|
215
|
+
{ error: e.class.name }
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
{ id: call.id, name: call.name, response: }
|
|
219
|
+
end
|
|
220
|
+
deliver(toolResponse: { functionResponses: responses })
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# A frame that fails to parse or build closes the session with an error naming only the exception class: the
|
|
224
|
+
# original's message (and so its cause) can quote the frame, which may hold transcripts.
|
|
225
|
+
#
|
|
226
|
+
# @param data [String]
|
|
227
|
+
#
|
|
228
|
+
# @raise [Error]
|
|
229
|
+
#
|
|
230
|
+
# @return [Array(Hash, Array)] the message and its events
|
|
231
|
+
def decode(data)
|
|
232
|
+
message = JSON.parse(data)
|
|
233
|
+
[message, message["setupComplete"] ? [] : events(message)]
|
|
234
|
+
rescue StandardError => e
|
|
235
|
+
raise Error.new("#{self.class.name} could not decode a server message: #{e.class.name} " \
|
|
236
|
+
"(session=#{@session_id || '-'})"), cause: nil
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# Server message keys and their event builders, in the order events are yielded (usage after content).
|
|
240
|
+
MESSAGE_EVENTS = {
|
|
241
|
+
"serverContent" => :server_content_events,
|
|
242
|
+
"toolCall" => :tool_call_events,
|
|
243
|
+
"toolCallCancellation" => :cancellation_events,
|
|
244
|
+
"voiceActivity" => :voice_activity_events,
|
|
245
|
+
"sessionResumptionUpdate" => :resumption_events,
|
|
246
|
+
"goAway" => :go_away_events,
|
|
247
|
+
"usageMetadata" => :usage_events,
|
|
248
|
+
}.freeze
|
|
249
|
+
private_constant :MESSAGE_EVENTS
|
|
250
|
+
|
|
251
|
+
# @param message [Hash]
|
|
252
|
+
# @return [Array]
|
|
253
|
+
def events(message)
|
|
254
|
+
MESSAGE_EVENTS.flat_map { |key, builder| message[key] ? send(builder, message[key]) : [] }
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def server_content_events(content)
|
|
258
|
+
[
|
|
259
|
+
*transcript_events(:user, content["inputTranscription"]),
|
|
260
|
+
*model_turn_events(content["modelTurn"]),
|
|
261
|
+
*transcript_events(:model, content["outputTranscription"]),
|
|
262
|
+
(Event::Interrupted.new if content["interrupted"]),
|
|
263
|
+
(Event::GenerationComplete.new if content["generationComplete"]),
|
|
264
|
+
(Event::TurnComplete.new if content["turnComplete"]),
|
|
265
|
+
].compact
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
def transcript_events(role, transcription)
|
|
269
|
+
return [] unless transcription
|
|
270
|
+
|
|
271
|
+
[Event::Transcript.new(role:, text: transcription["text"], finished: transcription["finished"] == true)]
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
def model_turn_events(turn)
|
|
275
|
+
(turn&.dig("parts") || []).filter_map do |part|
|
|
276
|
+
if (inline = part["inlineData"])
|
|
277
|
+
Event::Audio.new(data: Base64.decode64(inline["data"]), mime_type: inline["mimeType"])
|
|
278
|
+
elsif part["text"] && !part["thought"]
|
|
279
|
+
Event::Text.new(text: part["text"])
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
def tool_call_events(tool_call)
|
|
285
|
+
calls = (tool_call["functionCalls"] || []).map do |call|
|
|
286
|
+
Event::FunctionCall.new(id: call["id"], name: call["name"], arguments: call["args"] || {})
|
|
287
|
+
end
|
|
288
|
+
[Event::ToolCall.new(calls:)]
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def cancellation_events(cancellation)
|
|
292
|
+
[Event::ToolCallCancellation.new(ids: cancellation["ids"] || [])]
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
def voice_activity_events(activity)
|
|
296
|
+
type = ACTIVITY.fetch(activity["type"], activity["type"])
|
|
297
|
+
[Event::VoiceActivity.new(type:, offset: seconds(activity["audioOffset"]))]
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def resumption_events(update)
|
|
301
|
+
[Event::ResumptionUpdate.new(handle: update["newHandle"], resumable: update["resumable"] == true)]
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def go_away_events(go_away)
|
|
305
|
+
[Event::GoAway.new(time_left: seconds(go_away["timeLeft"]))]
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
def usage_events(usage)
|
|
309
|
+
[Event::Usage.new(usage: Chat::UsageSerializer.deserialize(usage), data: usage)]
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# @return [Hash]
|
|
313
|
+
def setup
|
|
314
|
+
{
|
|
315
|
+
setup: {
|
|
316
|
+
model: "projects/#{project_id}/locations/#{location_id}/publishers/google/models/#{@model}",
|
|
317
|
+
generationConfig: { responseModalities: ["AUDIO"], speechConfig: speech_config }.compact,
|
|
318
|
+
systemInstruction: ({ parts: [{ text: @system }] } if @system),
|
|
319
|
+
tools: ([{ functionDeclarations: @tools.map { |tool| declaration(tool) } }] if @tools.any?),
|
|
320
|
+
inputAudioTranscription: ({} if @transcription),
|
|
321
|
+
outputAudioTranscription: ({} if @transcription),
|
|
322
|
+
realtimeInputConfig: realtime_input_config,
|
|
323
|
+
contextWindowCompression: context_window_compression,
|
|
324
|
+
sessionResumption: session_resumption,
|
|
325
|
+
}.compact,
|
|
326
|
+
}
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
# @param tool [OmniAI::Tool]
|
|
330
|
+
# @return [Hash]
|
|
331
|
+
def declaration(tool)
|
|
332
|
+
behavior = @tool_behavior.is_a?(Hash) ? @tool_behavior.fetch(tool.name, :blocking) : @tool_behavior
|
|
333
|
+
Chat::ToolSerializer.serialize(tool).merge(behavior: BEHAVIORS.fetch(behavior.to_sym))
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
# @return [Hash, nil]
|
|
337
|
+
def speech_config
|
|
338
|
+
{ voiceConfig: { prebuiltVoiceConfig: { voiceName: @voice } } } if @voice
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
# @return [Hash, nil]
|
|
342
|
+
def realtime_input_config
|
|
343
|
+
return unless @vad
|
|
344
|
+
|
|
345
|
+
detection = {
|
|
346
|
+
disabled: @vad[:disabled],
|
|
347
|
+
startOfSpeechSensitivity: sensitivity("START", @vad[:start_sensitivity]),
|
|
348
|
+
endOfSpeechSensitivity: sensitivity("END", @vad[:end_sensitivity]),
|
|
349
|
+
prefixPaddingMs: @vad[:prefix_padding_ms],
|
|
350
|
+
silenceDurationMs: @vad[:silence_ms],
|
|
351
|
+
}.compact
|
|
352
|
+
{ automaticActivityDetection: detection }
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
# @return [String, nil] e.g. "START_SENSITIVITY_HIGH"
|
|
356
|
+
def sensitivity(edge, level)
|
|
357
|
+
"#{edge}_SENSITIVITY_#{level.to_s.upcase}" if level
|
|
358
|
+
end
|
|
359
|
+
|
|
360
|
+
# @return [Hash, nil]
|
|
361
|
+
def context_window_compression
|
|
362
|
+
return unless @compression
|
|
363
|
+
|
|
364
|
+
options = @compression.is_a?(Hash) ? @compression : {}
|
|
365
|
+
{
|
|
366
|
+
triggerTokens: options[:trigger_tokens],
|
|
367
|
+
slidingWindow: { targetTokens: options[:target_tokens] }.compact,
|
|
368
|
+
}.compact
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
# @return [Hash, nil]
|
|
372
|
+
def session_resumption
|
|
373
|
+
case @resumption
|
|
374
|
+
when true then {}
|
|
375
|
+
when String then { handle: @resumption }
|
|
376
|
+
end
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
# @param value [String, nil] e.g. "0.200s"
|
|
380
|
+
# @return [Float, nil]
|
|
381
|
+
def seconds(value)
|
|
382
|
+
value&.delete_suffix("s")&.to_f
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
# @return [String] e.g. "wss://aiplatform.us.rep.googleapis.com/ws/..."
|
|
386
|
+
def url
|
|
387
|
+
"#{host.sub(/\Ahttp/, 'ws')}#{PATH}"
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
# @return [String]
|
|
391
|
+
def host
|
|
392
|
+
@client.instance_variable_get(:@host)
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
# @return [String]
|
|
396
|
+
def project_id
|
|
397
|
+
@client.instance_variable_get(:@project_id) || raise(ArgumentError, "project_id is required for Live")
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
# @return [String]
|
|
401
|
+
def location_id
|
|
402
|
+
@client.instance_variable_get(:@location_id) || raise(ArgumentError, "location_id is required for Live")
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
# @param message [String] ids, codes and classes only
|
|
406
|
+
def log(message)
|
|
407
|
+
@client.logger&.info(message)
|
|
408
|
+
end
|
|
409
|
+
end
|
|
410
|
+
end
|
|
411
|
+
end
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OmniAI
|
|
4
|
+
module Google
|
|
5
|
+
class Transcribe
|
|
6
|
+
# Transcribes with a Gemini transcribe model (e.g. `gemini-3.5-transcribe`). Unlike Chirp (Cloud Speech-to-Text),
|
|
7
|
+
# these are served by Vertex `generateContent` with the audio as the only part.
|
|
8
|
+
#
|
|
9
|
+
# Usage:
|
|
10
|
+
#
|
|
11
|
+
# client.transcribe("interview.wav", model: "gemini-3.5-transcribe", diarization: true)
|
|
12
|
+
class Gemini
|
|
13
|
+
include TranscribeHelpers
|
|
14
|
+
|
|
15
|
+
INLINE_LIMIT = 10_000_000 # bytes; larger audio goes through GCS
|
|
16
|
+
|
|
17
|
+
# Leading bytes of the audio containers Gemini accepts. `generateContent` requires a mimeType.
|
|
18
|
+
SIGNATURES = {
|
|
19
|
+
"RIFF" => "audio/wav",
|
|
20
|
+
"ID3" => "audio/mpeg",
|
|
21
|
+
"\xFF\xFB".b => "audio/mpeg",
|
|
22
|
+
"\xFF\xF3".b => "audio/mpeg",
|
|
23
|
+
"\xFF\xF2".b => "audio/mpeg",
|
|
24
|
+
"fLaC" => "audio/flac",
|
|
25
|
+
"OggS" => "audio/ogg",
|
|
26
|
+
"\x1A\x45\xDF\xA3".b => "audio/webm",
|
|
27
|
+
"FORM" => "audio/aiff",
|
|
28
|
+
}.freeze
|
|
29
|
+
|
|
30
|
+
EXTENSIONS = {
|
|
31
|
+
".wav" => "audio/wav", ".mp3" => "audio/mpeg", ".flac" => "audio/flac", ".ogg" => "audio/ogg",
|
|
32
|
+
".opus" => "audio/ogg", ".webm" => "audio/webm", ".m4a" => "audio/mp4", ".mp4" => "audio/mp4",
|
|
33
|
+
".aac" => "audio/aac", ".aif" => "audio/aiff", ".aiff" => "audio/aiff",
|
|
34
|
+
}.freeze
|
|
35
|
+
|
|
36
|
+
# @param model [String]
|
|
37
|
+
# @return [Boolean]
|
|
38
|
+
def self.model?(model)
|
|
39
|
+
String(model).start_with?("gemini-")
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def self.process!(...)
|
|
43
|
+
new(...).process!
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @param io [String, Pathname, IO] a file path, IO or `gs://` URI
|
|
47
|
+
# @param client [OmniAI::Google::Client] must use Vertex credentials
|
|
48
|
+
# @param model [String]
|
|
49
|
+
# @param language [String, Array<String>, nil] optional - auto-detected when omitted
|
|
50
|
+
# @param format [String, nil] optional - `verbose_json` adds per-utterance segments
|
|
51
|
+
# @param diarization [Boolean] optional - speaker labels per segment (experimental)
|
|
52
|
+
# @param word_timestamps [Boolean] optional - word times (experimental; can drop speech, see README)
|
|
53
|
+
# @param vocabulary [Array<String>, nil] optional - custom vocabulary (up to 1,000 terms)
|
|
54
|
+
def initialize(io, client:, model:, language: nil, format: nil, diarization: false, word_timestamps: false,
|
|
55
|
+
vocabulary: nil)
|
|
56
|
+
@io = io
|
|
57
|
+
@client = client.for_model(model)
|
|
58
|
+
@model = model
|
|
59
|
+
@language = language
|
|
60
|
+
@format = format || OmniAI::Transcribe::Format::JSON
|
|
61
|
+
@diarization = diarization
|
|
62
|
+
@word_timestamps = word_timestamps
|
|
63
|
+
@vocabulary = vocabulary
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# @raise [OmniAI::HTTPError, OmniAI::Error]
|
|
67
|
+
#
|
|
68
|
+
# @return [OmniAI::Transcribe::Transcription]
|
|
69
|
+
def process!
|
|
70
|
+
raise ArgumentError, "#{@model} requires Vertex credentials" unless @client.credentials?
|
|
71
|
+
|
|
72
|
+
response = request!
|
|
73
|
+
raise HTTPError, response unless response.status.ok?
|
|
74
|
+
|
|
75
|
+
OmniAI::Transcribe::Transcription.parse(model: @model, format: @format, data: parse(response.parse))
|
|
76
|
+
ensure
|
|
77
|
+
cleanup_gcs_file(@uploaded_gcs_uri) if @uploaded_gcs_uri
|
|
78
|
+
@uploaded_gcs_uri = nil
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
# @return [HTTP::Response]
|
|
84
|
+
def request!
|
|
85
|
+
HTTP.persistent(host)
|
|
86
|
+
.timeout(http_timeout_options)
|
|
87
|
+
.accept(:json)
|
|
88
|
+
.auth(@client.send(:auth))
|
|
89
|
+
.post(path, json: payload)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Served from the global endpoint and the us/eu multi-regions, never a single region, so a zonal Vertex
|
|
93
|
+
# location_id maps to its multi-region (same rule as Chirp 3); `us` is the default.
|
|
94
|
+
#
|
|
95
|
+
# A location from `locations` is used as given.
|
|
96
|
+
#
|
|
97
|
+
# @return [String] "global", "us" or "eu", unless mapped
|
|
98
|
+
def gemini_location_id
|
|
99
|
+
return @client.instance_variable_get(:@location_id) if @client.located?
|
|
100
|
+
|
|
101
|
+
case @client.instance_variable_get(:@location_id)
|
|
102
|
+
when "global" then "global"
|
|
103
|
+
when /\A(eu|europe)/i then "eu"
|
|
104
|
+
else "us"
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# @return [String]
|
|
109
|
+
def host
|
|
110
|
+
Vertex.host(gemini_location_id)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# @return [String]
|
|
114
|
+
def path
|
|
115
|
+
"/v1/projects/#{project_id}/locations/#{gemini_location_id}/publishers/google/models/" \
|
|
116
|
+
"#{@model}:generateContent"
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# @return [Hash]
|
|
120
|
+
def payload
|
|
121
|
+
config = {
|
|
122
|
+
languageCodes: language_codes,
|
|
123
|
+
diarization: (true if @diarization),
|
|
124
|
+
wordTimestamp: (true if @word_timestamps),
|
|
125
|
+
customVocabulary: @vocabulary,
|
|
126
|
+
}.compact
|
|
127
|
+
|
|
128
|
+
payload = { contents: [{ role: "user", parts: [audio_part] }] }
|
|
129
|
+
payload[:generationConfig] = { audioTranscriptionConfig: config } if config.any?
|
|
130
|
+
payload
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# @return [Hash]
|
|
134
|
+
def audio_part
|
|
135
|
+
return file_data(@io, EXTENSIONS.fetch(File.extname(@io).downcase, "audio/wav")) if gcs_uri?
|
|
136
|
+
|
|
137
|
+
bytes = audio_bytes
|
|
138
|
+
mime_type = sniff(bytes)
|
|
139
|
+
return { inlineData: { mimeType: mime_type, data: Base64.strict_encode64(bytes) } } if inline?(bytes)
|
|
140
|
+
|
|
141
|
+
@uploaded_gcs_uri = Bucket.process!(client: @client, io: StringIO.new(bytes))
|
|
142
|
+
file_data(@uploaded_gcs_uri, mime_type)
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# @param bytes [String]
|
|
146
|
+
# @return [Boolean]
|
|
147
|
+
def inline?(bytes)
|
|
148
|
+
bytes.bytesize <= INLINE_LIMIT
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# @return [Hash]
|
|
152
|
+
def file_data(uri, mime_type)
|
|
153
|
+
{ fileData: { fileUri: uri, mimeType: mime_type } }
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# @return [Boolean]
|
|
157
|
+
def gcs_uri?
|
|
158
|
+
@io.is_a?(String) && @io.start_with?("gs://")
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# @return [String]
|
|
162
|
+
def audio_bytes
|
|
163
|
+
case @io
|
|
164
|
+
when String, Pathname then File.binread(@io)
|
|
165
|
+
when IO, StringIO
|
|
166
|
+
@io.rewind if @io.respond_to?(:rewind)
|
|
167
|
+
@io.read.b
|
|
168
|
+
else raise ArgumentError, "unsupported input type: #{@io.class}"
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# @param bytes [String]
|
|
173
|
+
# @return [String]
|
|
174
|
+
def sniff(bytes)
|
|
175
|
+
return "audio/mp4" if bytes.byteslice(4, 4) == "ftyp"
|
|
176
|
+
|
|
177
|
+
SIGNATURES.each { |signature, mime_type| return mime_type if bytes.start_with?(signature.b) }
|
|
178
|
+
raise ArgumentError, "unrecognised audio format (expected wav, mp3, flac, ogg, webm, m4a or aiff)"
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# @param data [Hash]
|
|
182
|
+
# @return [Hash]
|
|
183
|
+
def parse(data)
|
|
184
|
+
candidate = data.dig("candidates", 0) || {}
|
|
185
|
+
parts = candidate.dig("content", "parts")
|
|
186
|
+
if parts.nil? || parts.empty?
|
|
187
|
+
raise OmniAI::Error, "#{@model} returned no content (finish reason #{candidate['finishReason']})"
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
segments = parts.map.with_index { |part, index| segment(part, index) }
|
|
191
|
+
result = { "text" => segments.map { |segment| segment["text"] }.join(" ") }
|
|
192
|
+
result["segments"] = segments if @format == OmniAI::Transcribe::Format::VERBOSE_JSON
|
|
193
|
+
result
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# @param part [Hash]
|
|
197
|
+
# @param index [Integer]
|
|
198
|
+
# @return [Hash]
|
|
199
|
+
def segment(part, index)
|
|
200
|
+
transcription = part["audioTranscription"] || {}
|
|
201
|
+
words = transcription["words"]&.map do |word|
|
|
202
|
+
{ "word" => word["word"], "start" => parse_duration(word["startOffset"]),
|
|
203
|
+
"end" => parse_duration(word["endOffset"]), }
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
{
|
|
207
|
+
"id" => index,
|
|
208
|
+
"text" => transcription["text"] || part["text"],
|
|
209
|
+
"speaker" => transcription["speakerLabel"],
|
|
210
|
+
"start" => words&.first&.fetch("start"),
|
|
211
|
+
"end" => words&.last&.fetch("end"),
|
|
212
|
+
"words" => words,
|
|
213
|
+
}.compact
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
end
|
|
217
|
+
end
|
|
218
|
+
end
|
|
@@ -12,6 +12,7 @@ module OmniAI
|
|
|
12
12
|
include TranscribeHelpers
|
|
13
13
|
|
|
14
14
|
module Model
|
|
15
|
+
GEMINI_3_5_TRANSCRIBE = "gemini-3.5-transcribe"
|
|
15
16
|
CHIRP_3 = "chirp_3"
|
|
16
17
|
CHIRP_2 = "chirp_2"
|
|
17
18
|
CHIRP = "chirp"
|
|
@@ -32,6 +33,27 @@ module OmniAI
|
|
|
32
33
|
# No custom deserializers needed - let base class handle parsing
|
|
33
34
|
end
|
|
34
35
|
|
|
36
|
+
GEMINI_OPTIONS = %i[diarization word_timestamps vocabulary].freeze
|
|
37
|
+
|
|
38
|
+
# Gemini transcribe models (`gemini-*`) take a separate path; every Chirp / Speech-to-Text model is unchanged.
|
|
39
|
+
#
|
|
40
|
+
# @return [OmniAI::Transcribe::Transcription]
|
|
41
|
+
def self.process!(io, model:, **options)
|
|
42
|
+
return Gemini.process!(io, model:, **options) if Gemini.model?(model)
|
|
43
|
+
|
|
44
|
+
unsupported = options.keys & GEMINI_OPTIONS
|
|
45
|
+
if unsupported.any?
|
|
46
|
+
raise ArgumentError, "#{unsupported.join(', ')} only supported by Gemini transcribe models, not #{model}"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
super
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# `client` is swapped for `client.for_model(model)`, so `locations` routes the request.
|
|
53
|
+
def initialize(io, client:, model:, **)
|
|
54
|
+
super(io, client: client.for_model(model), model:, **)
|
|
55
|
+
end
|
|
56
|
+
|
|
35
57
|
# @raise [HTTPError]
|
|
36
58
|
#
|
|
37
59
|
# @return [OmniAI::Transcribe::Transcription]
|
|
@@ -12,8 +12,12 @@ module OmniAI
|
|
|
12
12
|
raise(ArgumentError, "project_id is required for transcription")
|
|
13
13
|
end
|
|
14
14
|
|
|
15
|
+
# A location from `locations` is used as given; otherwise it is derived per model.
|
|
16
|
+
#
|
|
15
17
|
# @return [String]
|
|
16
18
|
def location_id
|
|
19
|
+
return @client.instance_variable_get(:@location_id) if @client.located?
|
|
20
|
+
|
|
17
21
|
case @model
|
|
18
22
|
when "chirp_2"
|
|
19
23
|
"us-central1"
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module OmniAI
|
|
4
|
+
module Google
|
|
5
|
+
# Raised before any request when `locations` is configured but has no entry for the model.
|
|
6
|
+
class UnknownLocationError < OmniAI::Error
|
|
7
|
+
# @return [String]
|
|
8
|
+
attr_reader :model
|
|
9
|
+
|
|
10
|
+
# @param model [String]
|
|
11
|
+
def initialize(model)
|
|
12
|
+
@model = model
|
|
13
|
+
super("no location configured for model #{model.inspect}")
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|