omniai-google 3.19.0 → 3.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,411 @@
1
+ # frozen_string_literal: true
2
+
3
+ module OmniAI
4
+ module Google
5
+ # A Gemini Live API session on Vertex (BidiGenerateContent over a WebSocket): stream microphone audio in, receive
6
+ # speech, transcripts and tool calls back.
7
+ #
8
+ # Usage:
9
+ #
10
+ # session = client.live(model: "gemini-3.8-live", system: "...", tools: [RecordAnswer.new]) do |event|
11
+ # case event
12
+ # when OmniAI::Google::Live::Event::Audio then play(event.data)
13
+ # when OmniAI::Google::Live::Event::Closed then reconnect
14
+ # end
15
+ # end
16
+ # session.send_audio(pcm16_16khz)
17
+ # session.close
18
+ #
19
+ # Each session holds one socket and one reader thread. Events are yielded, and tools run, on that thread; the
20
+ # `send_*` methods may be called from any thread. The gem never logs or raises audio, transcripts or tool arguments.
21
+ class Live
22
+ # @!attribute code [Integer, nil] the WebSocket close code
23
+ # @!attribute reason [String, nil] Google's close reason; kept out of the message because it can echo content
24
+ class Error < OmniAI::Error
25
+ attr_reader :code, :reason
26
+
27
+ def initialize(message, code: nil, reason: nil)
28
+ super(message)
29
+ @code = code
30
+ @reason = reason
31
+ end
32
+ end
33
+
34
+ class ClosedError < Error; end
35
+ class TimeoutError < Error; end
36
+
37
+ module Model
38
+ GEMINI_3_8_LIVE = "gemini-3.8-live"
39
+ end
40
+
41
+ DEFAULT_MODEL = Model::GEMINI_3_8_LIVE
42
+ INPUT_MIME_TYPE = "audio/pcm;rate=16000"
43
+ PATH = "/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent"
44
+ SETUP_TIMEOUT = 15 # seconds
45
+ WRITE_TIMEOUT = 5 # seconds
46
+ BEHAVIORS = { blocking: "BLOCKING", non_blocking: "NON_BLOCKING" }.freeze
47
+ ACTIVITY = { "ACTIVITY_START" => :start, "ACTIVITY_END" => :end }.freeze
48
+
49
+ # Opens a session and waits for the server's setupComplete.
50
+ #
51
+ # @raise [Error] if the server rejects the setup or it times out
52
+ #
53
+ # @return [Live]
54
+ def self.open(...)
55
+ new(...).tap(&:open)
56
+ end
57
+
58
+ # @return [String, nil] the server's session id, once open
59
+ attr_reader :session_id
60
+
61
+ # @param client [OmniAI::Google::Client] must use Vertex credentials, a project and a location (e.g. "us")
62
+ # @param model [String]
63
+ # @param system [String, nil] the system instruction
64
+ # @param tools [Array<OmniAI::Tool>, nil] run automatically; each result is sent back as a tool response
65
+ # @param tool_behavior [Symbol, Hash] `:blocking` (default) or `:non_blocking`, or a Hash of tool name to either
66
+ # @param voice [String, nil] a prebuilt voice name, e.g. "Kore"
67
+ # @param transcription [Boolean] transcribe both the user's and the model's speech (default true)
68
+ # @param vad [Hash, nil] automatic activity detection: `disabled:`, `start_sensitivity:` / `end_sensitivity:`
69
+ # (`:high` or `:low`), `prefix_padding_ms:`, `silence_ms:`
70
+ # @param compression [Boolean, Hash] sliding-window context compression, which lifts the 15-minute session cap and
71
+ # bounds the context re-billed every turn. true uses Google's defaults (trigger at 80% of the context window);
72
+ # or pass `trigger_tokens:` / `target_tokens:`
73
+ # @param resumption [Boolean, String, nil] true to receive resumption handles, or a handle to resume
74
+ # @param write_timeout [Numeric] seconds a send may block before the session is closed
75
+ #
76
+ # @yieldparam event [Live::Event::*]
77
+ def initialize(client:, model: DEFAULT_MODEL, system: nil, tools: nil, tool_behavior: :blocking, voice: nil,
78
+ transcription: true, vad: nil, compression: false, resumption: nil, write_timeout: WRITE_TIMEOUT, &handler)
79
+ @client = client.for_model(model)
80
+ @model = model
81
+ @system = system
82
+ @tools = tools || []
83
+ @tool_behavior = tool_behavior
84
+ @voice = voice
85
+ @transcription = transcription
86
+ @vad = vad
87
+ @compression = compression
88
+ @resumption = resumption
89
+ @write_timeout = write_timeout
90
+ @handler = handler
91
+ @state = :new
92
+ end
93
+
94
+ # @raise [Error]
95
+ #
96
+ # @return [Live]
97
+ def open
98
+ raise ArgumentError, "#{self.class.name} requires Vertex credentials" unless @client.credentials?
99
+
100
+ @ready = Queue.new
101
+ @connection = Connection.new(
102
+ url:,
103
+ headers: { "Authorization" => @client.send(:auth) }, # a fresh token per connection
104
+ write_timeout: @write_timeout,
105
+ on_open: method(:on_open),
106
+ on_message: method(:on_message),
107
+ on_close: method(:on_close)
108
+ )
109
+ log("LIVE connect #{host} #{@model}")
110
+ begin
111
+ @connection.connect
112
+ rescue Connection::ConnectTimeout => e
113
+ raise TimeoutError, "#{self.class.name} #{e.message} (model=#{@model})"
114
+ end
115
+ await_setup!
116
+ self
117
+ end
118
+
119
+ # @return [Boolean]
120
+ def open?
121
+ @state == :open && @connection.open?
122
+ end
123
+
124
+ # @param audio [String] raw PCM16 audio, 16 kHz, mono, little-endian
125
+ def send_audio(audio)
126
+ deliver(realtimeInput: { audio: { mimeType: INPUT_MIME_TYPE, data: Base64.strict_encode64(audio) } })
127
+ end
128
+
129
+ # Tells the server the microphone paused, so it flushes any buffered audio.
130
+ def end_audio
131
+ deliver(realtimeInput: { audioStreamEnd: true })
132
+ end
133
+
134
+ # Sends a complete user turn of text, e.g. to re-seed context after reconnecting.
135
+ #
136
+ # @param text [String]
137
+ def send_text(text)
138
+ deliver(clientContent: { turns: [{ role: "user", parts: [{ text: }] }], turnComplete: true })
139
+ end
140
+
141
+ def close
142
+ @connection&.close
143
+ end
144
+
145
+ private
146
+
147
+ # @param message [Hash]
148
+ #
149
+ # @raise [ClosedError, TimeoutError]
150
+ def deliver(message)
151
+ raise ClosedError, "#{self.class.name} session is closed (session=#{@session_id})" unless open?
152
+
153
+ @connection.send_text(JSON.generate(message))
154
+ rescue Connection::WriteTimeout
155
+ raise TimeoutError, "#{self.class.name} write timed out after #{@write_timeout}s (session=#{@session_id})"
156
+ end
157
+
158
+ def await_setup!
159
+ result = @ready.pop(timeout: SETUP_TIMEOUT)
160
+ return if result == :ready
161
+
162
+ @connection.close
163
+ raise Error, "#{self.class.name} setup timed out (model=#{@model})" if result.nil?
164
+
165
+ code, reason, error = result
166
+ raise Error.new("#{self.class.name} setup failed (model=#{@model} code=#{code.inspect} " \
167
+ "error=#{error&.class.inspect})", code:, reason:)
168
+ end
169
+
170
+ def on_open
171
+ @connection.send_text(JSON.generate(setup))
172
+ end
173
+
174
+ # @param data [String]
175
+ def on_message(data)
176
+ message, events = decode(data)
177
+ if (complete = message["setupComplete"])
178
+ @session_id = complete["sessionId"]
179
+ @state = :open
180
+ @ready << :ready
181
+ return
182
+ end
183
+
184
+ events.each do |event|
185
+ @handler&.call(event)
186
+ respond(event) if event.is_a?(Event::ToolCall)
187
+ end
188
+ end
189
+
190
+ def on_close(code, reason, error)
191
+ state = @state
192
+ @state = :closed
193
+ log("LIVE close #{code || '-'} #{@session_id || '-'}")
194
+ if state == :open
195
+ @handler&.call(Event::Closed.new(code:, reason:, error:))
196
+ else
197
+ @ready << [code, reason, error]
198
+ end
199
+ end
200
+
201
+ # Runs each called tool and sends the results. A raising tool answers with its error class only (never its
202
+ # message, which may carry arguments), and the session carries on.
203
+ #
204
+ # @param event [Event::ToolCall]
205
+ def respond(event)
206
+ responses = event.calls.map do |call|
207
+ tool = @tools.find { |candidate| candidate.name == call.name }
208
+ response =
209
+ if tool.nil?
210
+ { error: "unknown tool: #{call.name}" }
211
+ else
212
+ begin
213
+ { output: tool.call(call.arguments) }
214
+ rescue StandardError => e
215
+ { error: e.class.name }
216
+ end
217
+ end
218
+ { id: call.id, name: call.name, response: }
219
+ end
220
+ deliver(toolResponse: { functionResponses: responses })
221
+ end
222
+
223
+ # A frame that fails to parse or build closes the session with an error naming only the exception class: the
224
+ # original's message (and so its cause) can quote the frame, which may hold transcripts.
225
+ #
226
+ # @param data [String]
227
+ #
228
+ # @raise [Error]
229
+ #
230
+ # @return [Array(Hash, Array)] the message and its events
231
+ def decode(data)
232
+ message = JSON.parse(data)
233
+ [message, message["setupComplete"] ? [] : events(message)]
234
+ rescue StandardError => e
235
+ raise Error.new("#{self.class.name} could not decode a server message: #{e.class.name} " \
236
+ "(session=#{@session_id || '-'})"), cause: nil
237
+ end
238
+
239
+ # Server message keys and their event builders, in the order events are yielded (usage after content).
240
+ MESSAGE_EVENTS = {
241
+ "serverContent" => :server_content_events,
242
+ "toolCall" => :tool_call_events,
243
+ "toolCallCancellation" => :cancellation_events,
244
+ "voiceActivity" => :voice_activity_events,
245
+ "sessionResumptionUpdate" => :resumption_events,
246
+ "goAway" => :go_away_events,
247
+ "usageMetadata" => :usage_events,
248
+ }.freeze
249
+ private_constant :MESSAGE_EVENTS
250
+
251
+ # @param message [Hash]
252
+ # @return [Array]
253
+ def events(message)
254
+ MESSAGE_EVENTS.flat_map { |key, builder| message[key] ? send(builder, message[key]) : [] }
255
+ end
256
+
257
+ def server_content_events(content)
258
+ [
259
+ *transcript_events(:user, content["inputTranscription"]),
260
+ *model_turn_events(content["modelTurn"]),
261
+ *transcript_events(:model, content["outputTranscription"]),
262
+ (Event::Interrupted.new if content["interrupted"]),
263
+ (Event::GenerationComplete.new if content["generationComplete"]),
264
+ (Event::TurnComplete.new if content["turnComplete"]),
265
+ ].compact
266
+ end
267
+
268
+ def transcript_events(role, transcription)
269
+ return [] unless transcription
270
+
271
+ [Event::Transcript.new(role:, text: transcription["text"], finished: transcription["finished"] == true)]
272
+ end
273
+
274
+ def model_turn_events(turn)
275
+ (turn&.dig("parts") || []).filter_map do |part|
276
+ if (inline = part["inlineData"])
277
+ Event::Audio.new(data: Base64.decode64(inline["data"]), mime_type: inline["mimeType"])
278
+ elsif part["text"] && !part["thought"]
279
+ Event::Text.new(text: part["text"])
280
+ end
281
+ end
282
+ end
283
+
284
+ def tool_call_events(tool_call)
285
+ calls = (tool_call["functionCalls"] || []).map do |call|
286
+ Event::FunctionCall.new(id: call["id"], name: call["name"], arguments: call["args"] || {})
287
+ end
288
+ [Event::ToolCall.new(calls:)]
289
+ end
290
+
291
+ def cancellation_events(cancellation)
292
+ [Event::ToolCallCancellation.new(ids: cancellation["ids"] || [])]
293
+ end
294
+
295
+ def voice_activity_events(activity)
296
+ type = ACTIVITY.fetch(activity["type"], activity["type"])
297
+ [Event::VoiceActivity.new(type:, offset: seconds(activity["audioOffset"]))]
298
+ end
299
+
300
+ def resumption_events(update)
301
+ [Event::ResumptionUpdate.new(handle: update["newHandle"], resumable: update["resumable"] == true)]
302
+ end
303
+
304
+ def go_away_events(go_away)
305
+ [Event::GoAway.new(time_left: seconds(go_away["timeLeft"]))]
306
+ end
307
+
308
+ def usage_events(usage)
309
+ [Event::Usage.new(usage: Chat::UsageSerializer.deserialize(usage), data: usage)]
310
+ end
311
+
312
+ # @return [Hash]
313
+ def setup
314
+ {
315
+ setup: {
316
+ model: "projects/#{project_id}/locations/#{location_id}/publishers/google/models/#{@model}",
317
+ generationConfig: { responseModalities: ["AUDIO"], speechConfig: speech_config }.compact,
318
+ systemInstruction: ({ parts: [{ text: @system }] } if @system),
319
+ tools: ([{ functionDeclarations: @tools.map { |tool| declaration(tool) } }] if @tools.any?),
320
+ inputAudioTranscription: ({} if @transcription),
321
+ outputAudioTranscription: ({} if @transcription),
322
+ realtimeInputConfig: realtime_input_config,
323
+ contextWindowCompression: context_window_compression,
324
+ sessionResumption: session_resumption,
325
+ }.compact,
326
+ }
327
+ end
328
+
329
+ # @param tool [OmniAI::Tool]
330
+ # @return [Hash]
331
+ def declaration(tool)
332
+ behavior = @tool_behavior.is_a?(Hash) ? @tool_behavior.fetch(tool.name, :blocking) : @tool_behavior
333
+ Chat::ToolSerializer.serialize(tool).merge(behavior: BEHAVIORS.fetch(behavior.to_sym))
334
+ end
335
+
336
+ # @return [Hash, nil]
337
+ def speech_config
338
+ { voiceConfig: { prebuiltVoiceConfig: { voiceName: @voice } } } if @voice
339
+ end
340
+
341
+ # @return [Hash, nil]
342
+ def realtime_input_config
343
+ return unless @vad
344
+
345
+ detection = {
346
+ disabled: @vad[:disabled],
347
+ startOfSpeechSensitivity: sensitivity("START", @vad[:start_sensitivity]),
348
+ endOfSpeechSensitivity: sensitivity("END", @vad[:end_sensitivity]),
349
+ prefixPaddingMs: @vad[:prefix_padding_ms],
350
+ silenceDurationMs: @vad[:silence_ms],
351
+ }.compact
352
+ { automaticActivityDetection: detection }
353
+ end
354
+
355
+ # @return [String, nil] e.g. "START_SENSITIVITY_HIGH"
356
+ def sensitivity(edge, level)
357
+ "#{edge}_SENSITIVITY_#{level.to_s.upcase}" if level
358
+ end
359
+
360
+ # @return [Hash, nil]
361
+ def context_window_compression
362
+ return unless @compression
363
+
364
+ options = @compression.is_a?(Hash) ? @compression : {}
365
+ {
366
+ triggerTokens: options[:trigger_tokens],
367
+ slidingWindow: { targetTokens: options[:target_tokens] }.compact,
368
+ }.compact
369
+ end
370
+
371
+ # @return [Hash, nil]
372
+ def session_resumption
373
+ case @resumption
374
+ when true then {}
375
+ when String then { handle: @resumption }
376
+ end
377
+ end
378
+
379
+ # @param value [String, nil] e.g. "0.200s"
380
+ # @return [Float, nil]
381
+ def seconds(value)
382
+ value&.delete_suffix("s")&.to_f
383
+ end
384
+
385
+ # @return [String] e.g. "wss://aiplatform.us.rep.googleapis.com/ws/..."
386
+ def url
387
+ "#{host.sub(/\Ahttp/, 'ws')}#{PATH}"
388
+ end
389
+
390
+ # @return [String]
391
+ def host
392
+ @client.instance_variable_get(:@host)
393
+ end
394
+
395
+ # @return [String]
396
+ def project_id
397
+ @client.instance_variable_get(:@project_id) || raise(ArgumentError, "project_id is required for Live")
398
+ end
399
+
400
+ # @return [String]
401
+ def location_id
402
+ @client.instance_variable_get(:@location_id) || raise(ArgumentError, "location_id is required for Live")
403
+ end
404
+
405
+ # @param message [String] ids, codes and classes only
406
+ def log(message)
407
+ @client.logger&.info(message)
408
+ end
409
+ end
410
+ end
411
+ end
@@ -0,0 +1,218 @@
1
+ # frozen_string_literal: true
2
+
3
+ module OmniAI
4
+ module Google
5
+ class Transcribe
6
+ # Transcribes with a Gemini transcribe model (e.g. `gemini-3.5-transcribe`). Unlike Chirp (Cloud Speech-to-Text),
7
+ # these are served by Vertex `generateContent` with the audio as the only part.
8
+ #
9
+ # Usage:
10
+ #
11
+ # client.transcribe("interview.wav", model: "gemini-3.5-transcribe", diarization: true)
12
+ class Gemini
13
+ include TranscribeHelpers
14
+
15
+ INLINE_LIMIT = 10_000_000 # bytes; larger audio goes through GCS
16
+
17
+ # Leading bytes of the audio containers Gemini accepts. `generateContent` requires a mimeType.
18
+ SIGNATURES = {
19
+ "RIFF" => "audio/wav",
20
+ "ID3" => "audio/mpeg",
21
+ "\xFF\xFB".b => "audio/mpeg",
22
+ "\xFF\xF3".b => "audio/mpeg",
23
+ "\xFF\xF2".b => "audio/mpeg",
24
+ "fLaC" => "audio/flac",
25
+ "OggS" => "audio/ogg",
26
+ "\x1A\x45\xDF\xA3".b => "audio/webm",
27
+ "FORM" => "audio/aiff",
28
+ }.freeze
29
+
30
+ EXTENSIONS = {
31
+ ".wav" => "audio/wav", ".mp3" => "audio/mpeg", ".flac" => "audio/flac", ".ogg" => "audio/ogg",
32
+ ".opus" => "audio/ogg", ".webm" => "audio/webm", ".m4a" => "audio/mp4", ".mp4" => "audio/mp4",
33
+ ".aac" => "audio/aac", ".aif" => "audio/aiff", ".aiff" => "audio/aiff",
34
+ }.freeze
35
+
36
+ # @param model [String]
37
+ # @return [Boolean]
38
+ def self.model?(model)
39
+ String(model).start_with?("gemini-")
40
+ end
41
+
42
+ def self.process!(...)
43
+ new(...).process!
44
+ end
45
+
46
+ # @param io [String, Pathname, IO] a file path, IO or `gs://` URI
47
+ # @param client [OmniAI::Google::Client] must use Vertex credentials
48
+ # @param model [String]
49
+ # @param language [String, Array<String>, nil] optional - auto-detected when omitted
50
+ # @param format [String, nil] optional - `verbose_json` adds per-utterance segments
51
+ # @param diarization [Boolean] optional - speaker labels per segment (experimental)
52
+ # @param word_timestamps [Boolean] optional - word times (experimental; can drop speech, see README)
53
+ # @param vocabulary [Array<String>, nil] optional - custom vocabulary (up to 1,000 terms)
54
+ def initialize(io, client:, model:, language: nil, format: nil, diarization: false, word_timestamps: false,
55
+ vocabulary: nil)
56
+ @io = io
57
+ @client = client.for_model(model)
58
+ @model = model
59
+ @language = language
60
+ @format = format || OmniAI::Transcribe::Format::JSON
61
+ @diarization = diarization
62
+ @word_timestamps = word_timestamps
63
+ @vocabulary = vocabulary
64
+ end
65
+
66
+ # @raise [OmniAI::HTTPError, OmniAI::Error]
67
+ #
68
+ # @return [OmniAI::Transcribe::Transcription]
69
+ def process!
70
+ raise ArgumentError, "#{@model} requires Vertex credentials" unless @client.credentials?
71
+
72
+ response = request!
73
+ raise HTTPError, response unless response.status.ok?
74
+
75
+ OmniAI::Transcribe::Transcription.parse(model: @model, format: @format, data: parse(response.parse))
76
+ ensure
77
+ cleanup_gcs_file(@uploaded_gcs_uri) if @uploaded_gcs_uri
78
+ @uploaded_gcs_uri = nil
79
+ end
80
+
81
+ private
82
+
83
+ # @return [HTTP::Response]
84
+ def request!
85
+ HTTP.persistent(host)
86
+ .timeout(http_timeout_options)
87
+ .accept(:json)
88
+ .auth(@client.send(:auth))
89
+ .post(path, json: payload)
90
+ end
91
+
92
+ # Served from the global endpoint and the us/eu multi-regions, never a single region, so a zonal Vertex
93
+ # location_id maps to its multi-region (same rule as Chirp 3); `us` is the default.
94
+ #
95
+ # A location from `locations` is used as given.
96
+ #
97
+ # @return [String] "global", "us" or "eu", unless mapped
98
+ def gemini_location_id
99
+ return @client.instance_variable_get(:@location_id) if @client.located?
100
+
101
+ case @client.instance_variable_get(:@location_id)
102
+ when "global" then "global"
103
+ when /\A(eu|europe)/i then "eu"
104
+ else "us"
105
+ end
106
+ end
107
+
108
+ # @return [String]
109
+ def host
110
+ Vertex.host(gemini_location_id)
111
+ end
112
+
113
+ # @return [String]
114
+ def path
115
+ "/v1/projects/#{project_id}/locations/#{gemini_location_id}/publishers/google/models/" \
116
+ "#{@model}:generateContent"
117
+ end
118
+
119
+ # @return [Hash]
120
+ def payload
121
+ config = {
122
+ languageCodes: language_codes,
123
+ diarization: (true if @diarization),
124
+ wordTimestamp: (true if @word_timestamps),
125
+ customVocabulary: @vocabulary,
126
+ }.compact
127
+
128
+ payload = { contents: [{ role: "user", parts: [audio_part] }] }
129
+ payload[:generationConfig] = { audioTranscriptionConfig: config } if config.any?
130
+ payload
131
+ end
132
+
133
+ # @return [Hash]
134
+ def audio_part
135
+ return file_data(@io, EXTENSIONS.fetch(File.extname(@io).downcase, "audio/wav")) if gcs_uri?
136
+
137
+ bytes = audio_bytes
138
+ mime_type = sniff(bytes)
139
+ return { inlineData: { mimeType: mime_type, data: Base64.strict_encode64(bytes) } } if inline?(bytes)
140
+
141
+ @uploaded_gcs_uri = Bucket.process!(client: @client, io: StringIO.new(bytes))
142
+ file_data(@uploaded_gcs_uri, mime_type)
143
+ end
144
+
145
+ # @param bytes [String]
146
+ # @return [Boolean]
147
+ def inline?(bytes)
148
+ bytes.bytesize <= INLINE_LIMIT
149
+ end
150
+
151
+ # @return [Hash]
152
+ def file_data(uri, mime_type)
153
+ { fileData: { fileUri: uri, mimeType: mime_type } }
154
+ end
155
+
156
+ # @return [Boolean]
157
+ def gcs_uri?
158
+ @io.is_a?(String) && @io.start_with?("gs://")
159
+ end
160
+
161
+ # @return [String]
162
+ def audio_bytes
163
+ case @io
164
+ when String, Pathname then File.binread(@io)
165
+ when IO, StringIO
166
+ @io.rewind if @io.respond_to?(:rewind)
167
+ @io.read.b
168
+ else raise ArgumentError, "unsupported input type: #{@io.class}"
169
+ end
170
+ end
171
+
172
+ # @param bytes [String]
173
+ # @return [String]
174
+ def sniff(bytes)
175
+ return "audio/mp4" if bytes.byteslice(4, 4) == "ftyp"
176
+
177
+ SIGNATURES.each { |signature, mime_type| return mime_type if bytes.start_with?(signature.b) }
178
+ raise ArgumentError, "unrecognised audio format (expected wav, mp3, flac, ogg, webm, m4a or aiff)"
179
+ end
180
+
181
+ # @param data [Hash]
182
+ # @return [Hash]
183
+ def parse(data)
184
+ candidate = data.dig("candidates", 0) || {}
185
+ parts = candidate.dig("content", "parts")
186
+ if parts.nil? || parts.empty?
187
+ raise OmniAI::Error, "#{@model} returned no content (finish reason #{candidate['finishReason']})"
188
+ end
189
+
190
+ segments = parts.map.with_index { |part, index| segment(part, index) }
191
+ result = { "text" => segments.map { |segment| segment["text"] }.join(" ") }
192
+ result["segments"] = segments if @format == OmniAI::Transcribe::Format::VERBOSE_JSON
193
+ result
194
+ end
195
+
196
+ # @param part [Hash]
197
+ # @param index [Integer]
198
+ # @return [Hash]
199
+ def segment(part, index)
200
+ transcription = part["audioTranscription"] || {}
201
+ words = transcription["words"]&.map do |word|
202
+ { "word" => word["word"], "start" => parse_duration(word["startOffset"]),
203
+ "end" => parse_duration(word["endOffset"]), }
204
+ end
205
+
206
+ {
207
+ "id" => index,
208
+ "text" => transcription["text"] || part["text"],
209
+ "speaker" => transcription["speakerLabel"],
210
+ "start" => words&.first&.fetch("start"),
211
+ "end" => words&.last&.fetch("end"),
212
+ "words" => words,
213
+ }.compact
214
+ end
215
+ end
216
+ end
217
+ end
218
+ end
@@ -12,6 +12,7 @@ module OmniAI
12
12
  include TranscribeHelpers
13
13
 
14
14
  module Model
15
+ GEMINI_3_5_TRANSCRIBE = "gemini-3.5-transcribe"
15
16
  CHIRP_3 = "chirp_3"
16
17
  CHIRP_2 = "chirp_2"
17
18
  CHIRP = "chirp"
@@ -32,6 +33,27 @@ module OmniAI
32
33
  # No custom deserializers needed - let base class handle parsing
33
34
  end
34
35
 
36
+ GEMINI_OPTIONS = %i[diarization word_timestamps vocabulary].freeze
37
+
38
+ # Gemini transcribe models (`gemini-*`) take a separate path; every Chirp / Speech-to-Text model is unchanged.
39
+ #
40
+ # @return [OmniAI::Transcribe::Transcription]
41
+ def self.process!(io, model:, **options)
42
+ return Gemini.process!(io, model:, **options) if Gemini.model?(model)
43
+
44
+ unsupported = options.keys & GEMINI_OPTIONS
45
+ if unsupported.any?
46
+ raise ArgumentError, "#{unsupported.join(', ')} only supported by Gemini transcribe models, not #{model}"
47
+ end
48
+
49
+ super
50
+ end
51
+
52
+ # `client` is swapped for `client.for_model(model)`, so `locations` routes the request.
53
+ def initialize(io, client:, model:, **)
54
+ super(io, client: client.for_model(model), model:, **)
55
+ end
56
+
35
57
  # @raise [HTTPError]
36
58
  #
37
59
  # @return [OmniAI::Transcribe::Transcription]
@@ -12,8 +12,12 @@ module OmniAI
12
12
  raise(ArgumentError, "project_id is required for transcription")
13
13
  end
14
14
 
15
+ # A location from `locations` is used as given; otherwise it is derived per model.
16
+ #
15
17
  # @return [String]
16
18
  def location_id
19
+ return @client.instance_variable_get(:@location_id) if @client.located?
20
+
17
21
  case @model
18
22
  when "chirp_2"
19
23
  "us-central1"
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module OmniAI
4
+ module Google
5
+ # Raised before any request when `locations` is configured but has no entry for the model.
6
+ class UnknownLocationError < OmniAI::Error
7
+ # @return [String]
8
+ attr_reader :model
9
+
10
+ # @param model [String]
11
+ def initialize(model)
12
+ @model = model
13
+ super("no location configured for model #{model.inspect}")
14
+ end
15
+ end
16
+ end
17
+ end