omniai-google 3.19.0 → 3.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e4c5367190abf998fe07c957ea0bcb092660540d96e6a81bbe82b1f67e751895
-  data.tar.gz: 1d37b3a65b76667d647f35091b432cfa4c94458d222a1d43120d509145045d94
+  metadata.gz: ce2434a17917341984b99049670ef65083443c2c04b8937d2b8d4af688a5b631
+  data.tar.gz: 6c22fa11d66efc7df6ca393bc7f167c5f682ac0398a934a6b25a8674920a6905
 SHA512:
-  metadata.gz: 4227aa6e92bc19605188ba27e0d5b3b33672f4d66b1cbfdc14a0d17c47ad205f86e893d131f9986c830df21135d778a97e962f1575c67adf5d72c5cac2d6b3ae
-  data.tar.gz: eb585c514a31d9fa8577968ff5e558a9b9da3762d7c316e4528dfecd7e60890f408adf5172d94a2b7981e3382d00c75177b3a2c6f092a371f1685037ec6dea06
+  metadata.gz: f935566b3cae177f1bd445f19e2a2b87ae32d9acc274d5a5a4cb470b867c36d3e2382d9ca945f5b8e5a00a617806a1c1d4399905b642c716b2c10e9bf6dfd568
+  data.tar.gz: b183524e4139f0a2470749fde7bb0fb837dc32cd0ca9f810af1559d44a2fbef4e62a3e5fe98e8b385de4f66559b55026a643a04371477f909ffd9bb1d45adaca

data/README.md

@@ -344,6 +344,36 @@ client.transcribe("audio.mp3", model: OmniAI::Google::Transcribe::Model::CHIRP)
 
 [Google Speech-to-Text API Reference](https://cloud.google.com/speech-to-text/docs)
 
+#### Gemini Transcribe
+
+`gemini-3.5-transcribe` is a Gemini model served by Vertex `generateContent`, not Speech-to-Text, so it needs Vertex
+credentials. Pass it as the model; the Chirp models above are unaffected.
+
+```ruby
+transcription = client.transcribe("interview.wav", model: OmniAI::Google::Transcribe::Model::GEMINI_3_5_TRANSCRIBE)
+transcription.text
+
+# Speaker labels per utterance (experimental, up to 8 speakers)
+transcription = client.transcribe("interview.wav", model: "gemini-3.5-transcribe", diarization: true, format: "verbose_json")
+transcription.segments # => [{ "id" => 0, "text" => "...", "speaker" => "spk:0" }, ...]
+
+# Language hints and custom vocabulary (up to 1,000 terms); the language is auto-detected when omitted
+client.transcribe("interview.wav", model: "gemini-3.5-transcribe", language: "es-US", vocabulary: %w[ibuprofeno])
+
+# Word times (experimental, off by default; see the caveat below)
+client.transcribe("interview.wav", model: "gemini-3.5-transcribe", word_timestamps: true, format: "verbose_json")
+```
+
+- **Location:** served from `global` and the `us` / `eu` multi-regions, never a single region. A zonal `location_id`
+  maps to its multi-region (`us` by default, `eu` for `eu*` / `europe*`); `global` is kept.
+- **Input:** wav, mp3, flac, ogg, webm, m4a or aiff (detected from the bytes), or a `gs://` URI. Files over 10 MB are
+  uploaded to the `<project_id>-speech-audio` bucket and deleted afterwards.
+- **Length:** Google caps a file at 15 minutes when diarization or timestamps are on, so split longer recordings.
+- **Word timestamps can drop speech.** In live tests on a two-speaker, 8-second clip (2026-10-15), `word_timestamps: true`
+  returned only the first word of the answer, and with `diarization: true` as well it dropped the whole first utterance.
+  Google labels both features experimental and warns that word timestamps degrade accuracy. `diarization:` alone
+  returned the complete transcript but no times.
+
 #### GCS Setup for Transcription
 
 For transcription to work properly with automatic file uploads, you need to set up Google Cloud Storage and configure the appropriate permissions.
@@ -426,6 +456,59 @@ client.transcribe("audio.mp3", bucket_name: "my-custom-audio-bucket")
 - Audio files are automatically deleted after successful transcription
 - If transcription fails, temporary files may remain and should be cleaned up manually
 
+### Live
+
+A Gemini Live session (Vertex `BidiGenerateContent` over a WebSocket): stream microphone audio in, get speech,
+transcripts and tool calls back. Vertex credentials only.
+
+```ruby
+client = OmniAI::Google::Client.new(
+  project_id: "...", credentials: "...",
+  location_id: "us", host: "https://aiplatform.us.rep.googleapis.com" # gemini-3.8-live: us, eu or us-central1 (not global)
+)
+
+session = client.live(
+  model: OmniAI::Google::Live::Model::GEMINI_3_8_LIVE,
+  system: "You are an intake assistant...",
+  tools: [RecordAnswer.new],       # OmniAI::Tool; run automatically, results sent back
+  tool_behavior: :blocking,        # default; or :non_blocking, or { "flag" => :non_blocking }
+  voice: "Kore",                   # optional
+  vad: { silence_ms: 800 },        # optional
+  compression: { trigger_tokens: 10_000, target_tokens: 5_000 } # optional (or true): see Limits
+) do |event|
+  case event
+  when OmniAI::Google::Live::Event::Audio then speaker.write(event.data) # PCM16, 24 kHz, mono
+  when OmniAI::Google::Live::Event::Transcript then caption(event.role, event.text) # :user / :model
+  when OmniAI::Google::Live::Event::Interrupted then speaker.flush
+  when OmniAI::Google::Live::Event::GoAway then reconnect_within(event.time_left)
+  when OmniAI::Google::Live::Event::Usage then track(event.usage, event.data) # data has per-modality counts
+  when OmniAI::Google::Live::Event::Closed then reconnect_or_finish(event.code, event.error)
+  end
+end
+
+session.send_audio(pcm16_16khz) # from any thread
+session.send_text("We are at question 12; answers so far: ...") # e.g. to re-seed after reconnecting
+session.close
+```
+
+- **Threads:** each session has one socket and one reader thread. Events are yielded, and tools run, on that thread;
+  `send_*` may be called from any thread. A send that blocks longer than `write_timeout:` (default 5 s) closes the
+  session and raises `Live::TimeoutError`, as does a connect (TCP + TLS) slower than 10 s. An idle connection is pinged
+  every 15 s; with no answer in 10 s it is dropped, so a silently dead path still ends the session. The session always
+  ends with exactly one `Event::Closed`, including when the socket drops or your handler raises (`event.error`).
+- **Tools:** a raising tool answers the model with its error class only, and the session carries on.
+- **Limits (Google):** a connection lasts about 10 minutes, and an audio-only session 15 minutes without
+  `compression:`. Measured live at `us` (2026-10-15): `GoAway` arrived at 9 min (`time_left` 30 s, not the 60 s
+  Google documents) and the server closed with 1000 at 9.5 min. Reconnect when `GoAway` arrives, not on a timer: the documented numbers are off.
+- **Cost:** every turn re-bills the whole context. `compression:` bounds it: in a live test with a 10,000-token trigger
+  and 5,000-token target, `promptTokenCount` dropped from about 8,500 to 3,400 whenever the next turn would pass the
+  trigger (a sawtooth instead of linear growth; 51% fewer prompt tokens over 12 turns). The oldest turns are dropped,
+  so keep anything the model must remember in your own state and re-send it when needed. The gem does not reconnect: open a new session and re-seed it
+  (`resumption:` takes Google's resumption handles if you prefer that).
+- **Privacy:** the gem never logs or raises audio, transcripts or tool arguments. `Live::Error#message` carries codes and
+  ids only; Google's close reason is on `#reason`, and `Event::Closed#inspect` leaves it out. A server frame that fails
+  to parse closes the session with a `Live::Error` naming only the exception class.
+
 ### Embed
 
 Text can be converted into a vector embedding for similarity comparison usage via:

data/lib/omniai/google/client.rb

@@ -96,8 +96,20 @@ module OmniAI
       # @param model [String] optional
       # @param language [String, Array<String>] optional - language codes for transcription
       # @param format [Symbol] optional - :json or :verbose_json
-      def transcribe(input, model: Transcribe::DEFAULT_MODEL, language: nil, format: nil)
-        Transcribe.process!(input, model:, language:, format:, client: self)
+      # @param options [Hash] Gemini transcribe models only: `diarization:`, `word_timestamps:`, `vocabulary:`
+      def transcribe(input, model: Transcribe::DEFAULT_MODEL, language: nil, format: nil, **)
+        Transcribe.process!(input, model:, language:, format:, client: self, **)
+      end
+
+      # Opens a Gemini Live session (Vertex only). See `OmniAI::Google::Live#initialize` for the options.
+      #
+      # @raise [OmniAI::Google::Live::Error]
+      #
+      # @yieldparam event [OmniAI::Google::Live::Event::*]
+      #
+      # @return [OmniAI::Google::Live]
+      def live(model: Live::DEFAULT_MODEL, **, &)
+        Live.open(client: self, model:, **, &)
       end
 
       # @return [String]

data/lib/omniai/google/embed.rb

@@ -155,7 +155,7 @@ module OmniAI
 
       # @return [String]
       def path
-        "/#{@client.path}/models/#{@model}:#{PROCEDURES[endpoint]}"
+        "#{@client.path}/models/#{@model}:#{PROCEDURES[endpoint]}"
       end
     end
   end

data/lib/omniai/google/live/connection.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+require "socket"
+require "websocket/driver"
+
+module OmniAI
+  module Google
+    class Live
+      # A WebSocket client over TCP or TLS, driven by websocket-driver with one reader thread per connection.
+      #
+      # Callbacks run on the reader thread. Writes may come from any thread: they are serialized by a mutex and bounded
+      # by `write_timeout`, so a stalled peer can't block the caller indefinitely. An idle connection is pinged, so a
+      # silently dropped path is noticed. `on_close` is called exactly once, whether the peer closes, the socket hits
+      # EOF or errors, a write or pong times out, or a callback raises.
+      class Connection
+        CONNECT_TIMEOUT = 10 # seconds, TCP and TLS together
+        CLOSE_TIMEOUT = 2 # seconds
+        PING_INTERVAL = 15 # seconds without a frame before pinging
+        PONG_TIMEOUT = 10 # seconds
+        READ_CHUNK = 16_384 # bytes
+
+        class ConnectTimeout < StandardError; end
+        class WriteTimeout < StandardError; end
+        class PongTimeout < StandardError; end
+
+        # @return [String]
+        attr_reader :url
+
+        # @param url [String] a ws:// or wss:// URL
+        # @param headers [Hash] sent with the upgrade request
+        # @param write_timeout [Numeric] seconds
+        # @param on_open [Proc] called with no arguments
+        # @param on_message [Proc] called with the payload String
+        # @param on_close [Proc] called with (code, reason, error)
+        def initialize(url:, headers:, write_timeout:, on_open:, on_message:, on_close:)
+          @url = url
+          @headers = headers
+          @write_timeout = write_timeout
+          @on_open = on_open
+          @on_message = on_message
+          @on_close = on_close
+          @mutex = Mutex.new
+          @pending = []
+          @finished = false
+        end
+
+        # @raise [ConnectTimeout]
+        def connect
+          @io = open_socket
+          @wake_reader, @wake_writer = IO.pipe # for abort!: on Linux, closing a socket doesn't interrupt a select on it
+          @driver = WebSocket::Driver.client(self, binary_data_format: :string)
+          @headers.each { |name, value| @driver.set_header(name, value) }
+          @driver.on(:open) { @pending << [:open] }
+          @driver.on(:message) { |event| @pending << [:message, event.data] }
+          @driver.on(:close) { |event| @pending << [:close, event.code, event.reason] }
+          @mutex.synchronize { @driver.start }
+          @reader = Thread.new { read_loop }
+          @reader.name = "omniai-google-live"
+          @reader.report_on_exception = false
+        rescue StandardError
+          [@io, @wake_reader, @wake_writer].compact.each { |io| close_quietly(io) } unless @reader
+          raise
+        end
+
+        # @return [Boolean]
+        def open?
+          !@finished && !@aborted
+        end
+
+        # @param payload [String]
+        #
+        # @raise [WriteTimeout]
+        def send_text(payload)
+          @mutex.synchronize { @driver.text(payload) }
+        rescue WriteTimeout => e
+          abort!(e)
+          raise
+        end
+
+        # Sends a close frame and waits briefly for the peer's reply before dropping the socket. Called from a callback
+        # (the reader thread itself), it only sends the frame: the reader finishes when the reply arrives.
+        def close
+          return if @finished || @reader.nil?
+
+          begin
+            @mutex.synchronize { @driver.close }
+          rescue WriteTimeout, IOError, SystemCallError
+            nil
+          end
+          return if Thread.current == @reader
+
+          abort! unless @reader.join(CLOSE_TIMEOUT)
+          @reader.join(CLOSE_TIMEOUT)
+        end
+
+        # Called by websocket-driver (under the mutex) for every outgoing frame.
+        #
+        # @param data [String]
+        def write(data)
+          deadline = monotonic + @write_timeout
+          data = data.b
+          until data.empty?
+            result = @io.write_nonblock(data, exception: false)
+            case result
+            when :wait_writable then wait!(nil, [@io.to_io], deadline) { write_timeout }
+            when :wait_readable then wait!([@io.to_io], nil, deadline) { write_timeout }
+            else data = data.byteslice(result..)
+            end
+          end
+        end
+
+      private
+
+        # @return [WriteTimeout]
+        def write_timeout
+          WriteTimeout.new("write timed out after #{@write_timeout}s")
+        end
+
+        # @return [Socket, OpenSSL::SSL::SSLSocket]
+        def open_socket
+          uri = URI.parse(@url)
+          deadline = monotonic + CONNECT_TIMEOUT
+          tcp = begin
+            Socket.tcp(uri.host, uri.port, connect_timeout: CONNECT_TIMEOUT)
+          rescue Errno::ETIMEDOUT, IO::TimeoutError
+            raise ConnectTimeout, "connect timed out after #{CONNECT_TIMEOUT}s"
+          end
+          return tcp unless uri.scheme == "wss"
+
+          handshake(tcp, uri.host, deadline)
+        end
+
+        # @return [OpenSSL::SSL::SSLSocket]
+        def handshake(tcp, host, deadline)
+          context = OpenSSL::SSL::SSLContext.new
+          context.set_params(verify_mode: OpenSSL::SSL::VERIFY_PEER)
+          ssl = OpenSSL::SSL::SSLSocket.new(tcp, context)
+          ssl.hostname = host
+          ssl.sync_close = true
+          timeout = -> { ConnectTimeout.new("connect timed out after #{CONNECT_TIMEOUT}s") }
+          loop do
+            case ssl.connect_nonblock(exception: false)
+            when :wait_readable then wait!([tcp], nil, deadline, &timeout)
+            when :wait_writable then wait!(nil, [tcp], deadline, &timeout)
+            else return ssl
+            end
+          end
+        rescue StandardError
+          tcp.close
+          raise
+        end
+
+        def read_loop
+          loop do
+            data = read_chunk
+            @ping_at = nil
+            events = @mutex.synchronize do
+              @driver.parse(data)
+              @pending.slice!(0..)
+            end
+            events.each { |event| dispatch(*event) }
+            break if @finished
+          end
+        rescue EOFError
+          finish(nil, nil, @abort_error)
+        rescue IOError, SystemCallError, OpenSSL::SSL::SSLError => e
+          finish(nil, nil, @abort_error || e)
+        rescue StandardError => e # a callback raised, or a ping went unanswered
+          finish(nil, nil, e)
+        ensure
+          finish(nil, nil, @abort_error)
+        end
+
+        # Waits for data, pinging when idle for PING_INTERVAL. Any frame (a pong included) counts as an answer.
+        #
+        # @raise [PongTimeout, EOFError]
+        #
+        # @return [String]
+        def read_chunk
+          loop do
+            result = @io.read_nonblock(READ_CHUNK, exception: false)
+            case result
+            when :wait_readable then idle([@io.to_io], nil)
+            when :wait_writable then idle(nil, [@io.to_io])
+            when nil then raise EOFError
+            else return result
+            end
+          end
+        end
+
+        def idle(readers, writers)
+          timeout = @ping_at ? @ping_at + PONG_TIMEOUT - monotonic : PING_INTERVAL
+          ready = IO.select([*readers, @wake_reader], writers, nil, timeout) if timeout.positive?
+          raise IOError, "aborted" if @aborted
+          return if ready
+
+          raise PongTimeout, "no pong within #{PONG_TIMEOUT}s" if @ping_at
+
+          @ping_at = monotonic
+          @mutex.synchronize { @driver.ping }
+        end
+
+        def dispatch(type, *args)
+          case type
+          when :open then @on_open.call
+          when :message then @on_message.call(args.first)
+          when :close then finish(args[0], args[1], nil)
+          end
+        end
+
+        # Drops the socket; the reader thread then finishes with `error`.
+        def abort!(error = nil)
+          @abort_error ||= error
+          @aborted = true
+          close_quietly(@io)
+          @wake_writer.write_nonblock(".", exception: false)
+        rescue IOError
+          nil # already finished
+        end
+
+        def finish(code, reason, error)
+          return if @finished
+
+          @finished = true
+          [@io, @wake_reader, @wake_writer].each { |io| close_quietly(io) }
+          begin
+            @on_close.call(code, reason, error)
+          rescue StandardError
+            nil
+          end
+        end
+
+        def close_quietly(io)
+          io.close
+        rescue IOError, SystemCallError
+          nil
+        end
+
+        # @yieldreturn [StandardError] raised once the deadline passes
+        def wait!(readers, writers, deadline)
+          remaining = deadline - monotonic
+          return if remaining.positive? && IO.select(readers, writers, nil, remaining)
+
+          raise yield
+        end
+
+        def monotonic
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+      end
+    end
+  end
+end

data/lib/omniai/google/live/event.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Google
+    class Live
+      # Events yielded by a Live session, on its reader thread.
+      module Event
+        # Model speech: raw PCM16 (24 kHz, mono, little-endian) bytes.
+        Audio = Data.define(:data, :mime_type) do
+          def inspect = "#<#{self.class.name} bytes=#{data.bytesize} mime_type=#{mime_type}>"
+        end
+
+        # Model text (only when the session asks for text responses).
+        Text = Data.define(:text)
+
+        # A transcript fragment of the user's (`:user`) or the model's (`:model`) speech.
+        Transcript = Data.define(:role, :text, :finished)
+
+        # @!attribute calls [Array<FunctionCall>]
+        ToolCall = Data.define(:calls)
+        FunctionCall = Data.define(:id, :name, :arguments)
+        ToolCallCancellation = Data.define(:ids)
+
+        # The user spoke over the model; drop any audio still queued for playback.
+        Interrupted = Data.define
+        GenerationComplete = Data.define
+        TurnComplete = Data.define
+
+        # Speech start/end detected by the server; `offset` is seconds into the session's input audio.
+        VoiceActivity = Data.define(:type, :offset)
+
+        # The server will end the connection in `time_left` seconds.
+        GoAway = Data.define(:time_left)
+        ResumptionUpdate = Data.define(:handle, :resumable)
+
+        # @!attribute usage [OmniAI::Chat::Usage] (`output_tokens` includes thinking)
+        # @!attribute data [Hash] the raw usageMetadata, with per-modality token details
+        Usage = Data.define(:usage, :data)
+
+        # The session ended. `code` is nil when the connection dropped without a close frame; `error` is set when it
+        # ended because of an exception (socket error, write timeout, or a raising handler). Google's close `reason`
+        # can echo request content, so `inspect` leaves it out.
+        Closed = Data.define(:code, :reason, :error) do
+          def inspect = "#<#{self.class.name} code=#{code.inspect} error=#{error&.class.inspect}>"
+        end
+      end
+    end
+  end
+end

data/lib/omniai/google/live.rb

@@ -0,0 +1,411 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Google
+    # A Gemini Live API session on Vertex (BidiGenerateContent over a WebSocket): stream microphone audio in, receive
+    # speech, transcripts and tool calls back.
+    #
+    # Usage:
+    #
+    #   session = client.live(model: "gemini-3.8-live", system: "...", tools: [RecordAnswer.new]) do |event|
+    #     case event
+    #     when OmniAI::Google::Live::Event::Audio then play(event.data)
+    #     when OmniAI::Google::Live::Event::Closed then reconnect
+    #     end
+    #   end
+    #   session.send_audio(pcm16_16khz)
+    #   session.close
+    #
+    # Each session holds one socket and one reader thread. Events are yielded, and tools run, on that thread; the
+    # `send_*` methods may be called from any thread. The gem never logs or raises audio, transcripts or tool arguments.
+    class Live
+      # @!attribute code [Integer, nil] the WebSocket close code
+      # @!attribute reason [String, nil] Google's close reason; kept out of the message because it can echo content
+      class Error < OmniAI::Error
+        attr_reader :code, :reason
+
+        def initialize(message, code: nil, reason: nil)
+          super(message)
+          @code = code
+          @reason = reason
+        end
+      end
+
+      class ClosedError < Error; end
+      class TimeoutError < Error; end
+
+      module Model
+        GEMINI_3_8_LIVE = "gemini-3.8-live"
+      end
+
+      DEFAULT_MODEL = Model::GEMINI_3_8_LIVE
+      INPUT_MIME_TYPE = "audio/pcm;rate=16000"
+      PATH = "/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent"
+      SETUP_TIMEOUT = 15 # seconds
+      WRITE_TIMEOUT = 5 # seconds
+      BEHAVIORS = { blocking: "BLOCKING", non_blocking: "NON_BLOCKING" }.freeze
+      ACTIVITY = { "ACTIVITY_START" => :start, "ACTIVITY_END" => :end }.freeze
+
+      # Opens a session and waits for the server's setupComplete.
+      #
+      # @raise [Error] if the server rejects the setup or it times out
+      #
+      # @return [Live]
+      def self.open(...)
+        new(...).tap(&:open)
+      end
+
+      # @return [String, nil] the server's session id, once open
+      attr_reader :session_id
+
+      # @param client [OmniAI::Google::Client] must use Vertex credentials, a project and a location (e.g. "us")
+      # @param model [String]
+      # @param system [String, nil] the system instruction
+      # @param tools [Array<OmniAI::Tool>, nil] run automatically; each result is sent back as a tool response
+      # @param tool_behavior [Symbol, Hash] `:blocking` (default) or `:non_blocking`, or a Hash of tool name to either
+      # @param voice [String, nil] a prebuilt voice name, e.g. "Kore"
+      # @param transcription [Boolean] transcribe both the user's and the model's speech (default true)
+      # @param vad [Hash, nil] automatic activity detection: `disabled:`, `start_sensitivity:` / `end_sensitivity:`
+      #   (`:high` or `:low`), `prefix_padding_ms:`, `silence_ms:`
+      # @param compression [Boolean, Hash] sliding-window context compression, which lifts the 15-minute session cap and
+      #   bounds the context re-billed every turn. true uses Google's defaults (trigger at 80% of the context window);
+      #   or pass `trigger_tokens:` / `target_tokens:`
+      # @param resumption [Boolean, String, nil] true to receive resumption handles, or a handle to resume
+      # @param write_timeout [Numeric] seconds a send may block before the session is closed
+      #
+      # @yieldparam event [Live::Event::*]
+      def initialize(client:, model: DEFAULT_MODEL, system: nil, tools: nil, tool_behavior: :blocking, voice: nil,
+        transcription: true, vad: nil, compression: false, resumption: nil, write_timeout: WRITE_TIMEOUT, &handler)
+        @client = client
+        @model = model
+        @system = system
+        @tools = tools || []
+        @tool_behavior = tool_behavior
+        @voice = voice
+        @transcription = transcription
+        @vad = vad
+        @compression = compression
+        @resumption = resumption
+        @write_timeout = write_timeout
+        @handler = handler
+        @state = :new
+      end
+
+      # @raise [Error]
+      #
+      # @return [Live]
+      def open
+        raise ArgumentError, "#{self.class.name} requires Vertex credentials" unless @client.credentials?
+
+        @ready = Queue.new
+        @connection = Connection.new(
+          url:,
+          headers: { "Authorization" => @client.send(:auth) }, # a fresh token per connection
+          write_timeout: @write_timeout,
+          on_open: method(:on_open),
+          on_message: method(:on_message),
+          on_close: method(:on_close)
+        )
+        log("LIVE connect #{host} #{@model}")
+        begin
+          @connection.connect
+        rescue Connection::ConnectTimeout => e
+          raise TimeoutError, "#{self.class.name} #{e.message} (model=#{@model})"
+        end
+        await_setup!
+        self
+      end
+
+      # @return [Boolean]
+      def open?
+        @state == :open && @connection.open?
+      end
+
+      # @param audio [String] raw PCM16 audio, 16 kHz, mono, little-endian
+      def send_audio(audio)
+        deliver(realtimeInput: { audio: { mimeType: INPUT_MIME_TYPE, data: Base64.strict_encode64(audio) } })
+      end
+
+      # Tells the server the microphone paused, so it flushes any buffered audio.
+      def end_audio
+        deliver(realtimeInput: { audioStreamEnd: true })
+      end
+
+      # Sends a complete user turn of text, e.g. to re-seed context after reconnecting.
+      #
+      # @param text [String]
+      def send_text(text)
+        deliver(clientContent: { turns: [{ role: "user", parts: [{ text: }] }], turnComplete: true })
+      end
+
+      def close
+        @connection&.close
+      end
+
+    private
+
+      # @param message [Hash]
+      #
+      # @raise [ClosedError, TimeoutError]
+      def deliver(message)
+        raise ClosedError, "#{self.class.name} session is closed (session=#{@session_id})" unless open?
+
+        @connection.send_text(JSON.generate(message))
+      rescue Connection::WriteTimeout
+        raise TimeoutError, "#{self.class.name} write timed out after #{@write_timeout}s (session=#{@session_id})"
+      end
+
+      def await_setup!
+        result = @ready.pop(timeout: SETUP_TIMEOUT)
+        return if result == :ready
+
+        @connection.close
+        raise Error, "#{self.class.name} setup timed out (model=#{@model})" if result.nil?
+
+        code, reason, error = result
+        raise Error.new("#{self.class.name} setup failed (model=#{@model} code=#{code.inspect} " \
+          "error=#{error&.class.inspect})", code:, reason:)
+      end
+
+      def on_open
+        @connection.send_text(JSON.generate(setup))
+      end
+
+      # @param data [String]
+      def on_message(data)
+        message, events = decode(data)
+        if (complete = message["setupComplete"])
+          @session_id = complete["sessionId"]
+          @state = :open
+          @ready << :ready
+          return
+        end
+
+        events.each do |event|
+          @handler&.call(event)
+          respond(event) if event.is_a?(Event::ToolCall)
+        end
+      end
+
+      def on_close(code, reason, error)
+        state = @state
+        @state = :closed
+        log("LIVE close #{code || '-'} #{@session_id || '-'}")
+        if state == :open
+          @handler&.call(Event::Closed.new(code:, reason:, error:))
+        else
+          @ready << [code, reason, error]
+        end
+      end
+
+      # Runs each called tool and sends the results. A raising tool answers with its error class only (never its
+      # message, which may carry arguments), and the session carries on.
+      #
+      # @param event [Event::ToolCall]
+      def respond(event)
+        responses = event.calls.map do |call|
+          tool = @tools.find { |candidate| candidate.name == call.name }
+          response =
+            if tool.nil?
+              { error: "unknown tool: #{call.name}" }
+            else
+              begin
+                { output: tool.call(call.arguments) }
+              rescue StandardError => e
+                { error: e.class.name }
+              end
+            end
+          { id: call.id, name: call.name, response: }
+        end
+        deliver(toolResponse: { functionResponses: responses })
+      end
+
+      # A frame that fails to parse or build closes the session with an error naming only the exception class: the
+      # original's message (and so its cause) can quote the frame, which may hold transcripts.
+      #
+      # @param data [String]
+      #
+      # @raise [Error]
+      #
+      # @return [Array(Hash, Array)] the message and its events
+      def decode(data)
+        message = JSON.parse(data)
+        [message, message["setupComplete"] ? [] : events(message)]
+      rescue StandardError => e
+        raise Error.new("#{self.class.name} could not decode a server message: #{e.class.name} " \
+          "(session=#{@session_id || '-'})"), cause: nil
+      end
+
+      # Server message keys and their event builders, in the order events are yielded (usage after content).
+      MESSAGE_EVENTS = {
+        "serverContent" => :server_content_events,
+        "toolCall" => :tool_call_events,
+        "toolCallCancellation" => :cancellation_events,
+        "voiceActivity" => :voice_activity_events,
+        "sessionResumptionUpdate" => :resumption_events,
+        "goAway" => :go_away_events,
+        "usageMetadata" => :usage_events,
+      }.freeze
+      private_constant :MESSAGE_EVENTS
+
+      # @param message [Hash]
+      # @return [Array]
+      def events(message)
+        MESSAGE_EVENTS.flat_map { |key, builder| message[key] ? send(builder, message[key]) : [] }
+      end
+
+      def server_content_events(content)
+        [
+          *transcript_events(:user, content["inputTranscription"]),
+          *model_turn_events(content["modelTurn"]),
+          *transcript_events(:model, content["outputTranscription"]),
+          (Event::Interrupted.new if content["interrupted"]),
+          (Event::GenerationComplete.new if content["generationComplete"]),
+          (Event::TurnComplete.new if content["turnComplete"]),
+        ].compact
+      end
+
+      def transcript_events(role, transcription)
+        return [] unless transcription
+
+        [Event::Transcript.new(role:, text: transcription["text"], finished: transcription["finished"] == true)]
+      end
+
+      def model_turn_events(turn)
+        (turn&.dig("parts") || []).filter_map do |part|
+          if (inline = part["inlineData"])
+            Event::Audio.new(data: Base64.decode64(inline["data"]), mime_type: inline["mimeType"])
+          elsif part["text"] && !part["thought"]
+            Event::Text.new(text: part["text"])
+          end
+        end
+      end
+
+      def tool_call_events(tool_call)
+        calls = (tool_call["functionCalls"] || []).map do |call|
+          Event::FunctionCall.new(id: call["id"], name: call["name"], arguments: call["args"] || {})
+        end
+        [Event::ToolCall.new(calls:)]
+      end
+
+      def cancellation_events(cancellation)
+        [Event::ToolCallCancellation.new(ids: cancellation["ids"] || [])]
+      end
+
+      def voice_activity_events(activity)
+        type = ACTIVITY.fetch(activity["type"], activity["type"])
+        [Event::VoiceActivity.new(type:, offset: seconds(activity["audioOffset"]))]
+      end
+
+      def resumption_events(update)
+        [Event::ResumptionUpdate.new(handle: update["newHandle"], resumable: update["resumable"] == true)]
+      end
+
+      def go_away_events(go_away)
+        [Event::GoAway.new(time_left: seconds(go_away["timeLeft"]))]
+      end
+
+      def usage_events(usage)
+        [Event::Usage.new(usage: Chat::UsageSerializer.deserialize(usage), data: usage)]
+      end
+
+      # @return [Hash]
+      def setup
+        {
+          setup: {
+            model: "projects/#{project_id}/locations/#{location_id}/publishers/google/models/#{@model}",
+            generationConfig: { responseModalities: ["AUDIO"], speechConfig: speech_config }.compact,
+            systemInstruction: ({ parts: [{ text: @system }] } if @system),
+            tools: ([{ functionDeclarations: @tools.map { |tool| declaration(tool) } }] if @tools.any?),
+            inputAudioTranscription: ({} if @transcription),
+            outputAudioTranscription: ({} if @transcription),
+            realtimeInputConfig: realtime_input_config,
+            contextWindowCompression: context_window_compression,
+            sessionResumption: session_resumption,
+          }.compact,
+        }
+      end
+
+      # @param tool [OmniAI::Tool]
+      # @return [Hash]
+      def declaration(tool)
+        behavior = @tool_behavior.is_a?(Hash) ? @tool_behavior.fetch(tool.name, :blocking) : @tool_behavior
+        Chat::ToolSerializer.serialize(tool).merge(behavior: BEHAVIORS.fetch(behavior.to_sym))
+      end
+
+      # @return [Hash, nil]
+      def speech_config
+        { voiceConfig: { prebuiltVoiceConfig: { voiceName: @voice } } } if @voice
+      end
+
+      # @return [Hash, nil]
+      def realtime_input_config
+        return unless @vad
+
+        detection = {
+          disabled: @vad[:disabled],
+          startOfSpeechSensitivity: sensitivity("START", @vad[:start_sensitivity]),
+          endOfSpeechSensitivity: sensitivity("END", @vad[:end_sensitivity]),
+          prefixPaddingMs: @vad[:prefix_padding_ms],
+          silenceDurationMs: @vad[:silence_ms],
+        }.compact
+        { automaticActivityDetection: detection }
+      end
+
+      # @return [String, nil] e.g. "START_SENSITIVITY_HIGH"
+      def sensitivity(edge, level)
+        "#{edge}_SENSITIVITY_#{level.to_s.upcase}" if level
+      end
+
+      # @return [Hash, nil]
+      def context_window_compression
+        return unless @compression
+
+        options = @compression.is_a?(Hash) ? @compression : {}
+        {
+          triggerTokens: options[:trigger_tokens],
+          slidingWindow: { targetTokens: options[:target_tokens] }.compact,
+        }.compact
+      end
+
+      # @return [Hash, nil]
+      def session_resumption
+        case @resumption
+        when true then {}
+        when String then { handle: @resumption }
+        end
+      end
+
+      # @param value [String, nil] e.g. "0.200s"
+      # @return [Float, nil]
+      def seconds(value)
+        value&.delete_suffix("s")&.to_f
+      end
+
+      # @return [String] e.g. "wss://aiplatform.us.rep.googleapis.com/ws/..."
+      def url
+        "#{host.sub(/\Ahttp/, 'ws')}#{PATH}"
+      end
+
+      # @return [String]
+      def host
+        @client.instance_variable_get(:@host)
+      end
+
+      # @return [String]
+      def project_id
+        @client.instance_variable_get(:@project_id) || raise(ArgumentError, "project_id is required for Live")
+      end
+
+      # @return [String]
+      def location_id
+        @client.instance_variable_get(:@location_id) || raise(ArgumentError, "location_id is required for Live")
+      end
+
+      # @param message [String] ids, codes and classes only
+      def log(message)
+        @client.logger&.info(message)
+      end
+    end
+  end
+end

data/lib/omniai/google/transcribe/gemini.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+module OmniAI
+  module Google
+    class Transcribe
+      # Transcribes with a Gemini transcribe model (e.g. `gemini-3.5-transcribe`). Unlike Chirp (Cloud Speech-to-Text),
+      # these are served by Vertex `generateContent` with the audio as the only part.
+      #
+      # Usage:
+      #
+      #   client.transcribe("interview.wav", model: "gemini-3.5-transcribe", diarization: true)
+      class Gemini
+        include TranscribeHelpers
+
+        INLINE_LIMIT = 10_000_000 # bytes; larger audio goes through GCS
+
+        # Leading bytes of the audio containers Gemini accepts. `generateContent` requires a mimeType.
+        SIGNATURES = {
+          "RIFF" => "audio/wav",
+          "ID3" => "audio/mpeg",
+          "\xFF\xFB".b => "audio/mpeg",
+          "\xFF\xF3".b => "audio/mpeg",
+          "\xFF\xF2".b => "audio/mpeg",
+          "fLaC" => "audio/flac",
+          "OggS" => "audio/ogg",
+          "\x1A\x45\xDF\xA3".b => "audio/webm",
+          "FORM" => "audio/aiff",
+        }.freeze
+
+        EXTENSIONS = {
+          ".wav" => "audio/wav", ".mp3" => "audio/mpeg", ".flac" => "audio/flac", ".ogg" => "audio/ogg",
+          ".opus" => "audio/ogg", ".webm" => "audio/webm", ".m4a" => "audio/mp4", ".mp4" => "audio/mp4",
+          ".aac" => "audio/aac", ".aif" => "audio/aiff", ".aiff" => "audio/aiff",
+        }.freeze
+
+        # @param model [String]
+        # @return [Boolean]
+        def self.model?(model)
+          String(model).start_with?("gemini-")
+        end
+
+        def self.process!(...)
+          new(...).process!
+        end
+
+        # @param io [String, Pathname, IO] a file path, IO or `gs://` URI
+        # @param client [OmniAI::Google::Client] must use Vertex credentials
+        # @param model [String]
+        # @param language [String, Array<String>, nil] optional - auto-detected when omitted
+        # @param format [String, nil] optional - `verbose_json` adds per-utterance segments
+        # @param diarization [Boolean] optional - speaker labels per segment (experimental)
+        # @param word_timestamps [Boolean] optional - word times (experimental; can drop speech, see README)
+        # @param vocabulary [Array<String>, nil] optional - custom vocabulary (up to 1,000 terms)
+        def initialize(io, client:, model:, language: nil, format: nil, diarization: false, word_timestamps: false,
+          vocabulary: nil)
+          @io = io
+          @client = client
+          @model = model
+          @language = language
+          @format = format || OmniAI::Transcribe::Format::JSON
+          @diarization = diarization
+          @word_timestamps = word_timestamps
+          @vocabulary = vocabulary
+        end
+
+        # @raise [OmniAI::HTTPError, OmniAI::Error]
+        #
+        # @return [OmniAI::Transcribe::Transcription]
+        def process!
+          raise ArgumentError, "#{@model} requires Vertex credentials" unless @client.credentials?
+
+          response = request!
+          raise HTTPError, response unless response.status.ok?
+
+          OmniAI::Transcribe::Transcription.parse(model: @model, format: @format, data: parse(response.parse))
+        ensure
+          cleanup_gcs_file(@uploaded_gcs_uri) if @uploaded_gcs_uri
+          @uploaded_gcs_uri = nil
+        end
+
+      private
+
+        # @return [HTTP::Response]
+        def request!
+          HTTP.persistent(host)
+            .timeout(http_timeout_options)
+            .accept(:json)
+            .auth(@client.send(:auth))
+            .post(path, json: payload)
+        end
+
+        # Served from the global endpoint and the us/eu multi-regions, never a single region, so a zonal Vertex
+        # location_id maps to its multi-region (same rule as Chirp 3); `us` is the default.
+        #
+        # @return [String] "global", "us" or "eu"
+        def gemini_location_id
+          case @client.instance_variable_get(:@location_id)
+          when "global" then "global"
+          when /\A(eu|europe)/i then "eu"
+          else "us"
+          end
+        end
+
+        # @return [String]
+        def host
+          location = gemini_location_id
+          location == "global" ? "https://aiplatform.googleapis.com" : "https://aiplatform.#{location}.rep.googleapis.com"
+        end
+
+        # @return [String]
+        def path
+          "/v1/projects/#{project_id}/locations/#{gemini_location_id}/publishers/google/models/" \
+            "#{@model}:generateContent"
+        end
+
+        # @return [Hash]
+        def payload
+          config = {
+            languageCodes: language_codes,
+            diarization: (true if @diarization),
+            wordTimestamp: (true if @word_timestamps),
+            customVocabulary: @vocabulary,
+          }.compact
+
+          payload = { contents: [{ role: "user", parts: [audio_part] }] }
+          payload[:generationConfig] = { audioTranscriptionConfig: config } if config.any?
+          payload
+        end
+
+        # @return [Hash]
+        def audio_part
+          return file_data(@io, EXTENSIONS.fetch(File.extname(@io).downcase, "audio/wav")) if gcs_uri?
+
+          bytes = audio_bytes
+          mime_type = sniff(bytes)
+          return { inlineData: { mimeType: mime_type, data: Base64.strict_encode64(bytes) } } if inline?(bytes)
+
+          @uploaded_gcs_uri = Bucket.process!(client: @client, io: StringIO.new(bytes))
+          file_data(@uploaded_gcs_uri, mime_type)
+        end
+
+        # @param bytes [String]
+        # @return [Boolean]
+        def inline?(bytes)
+          bytes.bytesize <= INLINE_LIMIT
+        end
+
+        # @return [Hash]
+        def file_data(uri, mime_type)
+          { fileData: { fileUri: uri, mimeType: mime_type } }
+        end
+
+        # @return [Boolean]
+        def gcs_uri?
+          @io.is_a?(String) && @io.start_with?("gs://")
+        end
+
+        # @return [String]
+        def audio_bytes
+          case @io
+          when String, Pathname then File.binread(@io)
+          when IO, StringIO
+            @io.rewind if @io.respond_to?(:rewind)
+            @io.read.b
+          else raise ArgumentError, "unsupported input type: #{@io.class}"
+          end
+        end
+
+        # @param bytes [String]
+        # @return [String]
+        def sniff(bytes)
+          return "audio/mp4" if bytes.byteslice(4, 4) == "ftyp"
+
+          SIGNATURES.each { |signature, mime_type| return mime_type if bytes.start_with?(signature.b) }
+          raise ArgumentError, "unrecognised audio format (expected wav, mp3, flac, ogg, webm, m4a or aiff)"
+        end
+
+        # @param data [Hash]
+        # @return [Hash]
+        def parse(data)
+          candidate = data.dig("candidates", 0) || {}
+          parts = candidate.dig("content", "parts")
+          if parts.nil? || parts.empty?
+            raise OmniAI::Error, "#{@model} returned no content (finish reason #{candidate['finishReason']})"
+          end
+
+          segments = parts.map.with_index { |part, index| segment(part, index) }
+          result = { "text" => segments.map { |segment| segment["text"] }.join(" ") }
+          result["segments"] = segments if @format == OmniAI::Transcribe::Format::VERBOSE_JSON
+          result
+        end
+
+        # @param part [Hash]
+        # @param index [Integer]
+        # @return [Hash]
+        def segment(part, index)
+          transcription = part["audioTranscription"] || {}
+          words = transcription["words"]&.map do |word|
+            { "word" => word["word"], "start" => parse_duration(word["startOffset"]),
+              "end" => parse_duration(word["endOffset"]), }
+          end
+
+          {
+            "id" => index,
+            "text" => transcription["text"] || part["text"],
+            "speaker" => transcription["speakerLabel"],
+            "start" => words&.first&.fetch("start"),
+            "end" => words&.last&.fetch("end"),
+            "words" => words,
+          }.compact
+        end
+      end
+    end
+  end
+end

data/lib/omniai/google/transcribe.rb

@@ -12,6 +12,7 @@ module OmniAI
       include TranscribeHelpers
 
       module Model
+        GEMINI_3_5_TRANSCRIBE = "gemini-3.5-transcribe"
         CHIRP_3 = "chirp_3"
         CHIRP_2 = "chirp_2"
         CHIRP = "chirp"
@@ -32,6 +33,22 @@ module OmniAI
         # No custom deserializers needed - let base class handle parsing
       end
 
+      GEMINI_OPTIONS = %i[diarization word_timestamps vocabulary].freeze
+
+      # Gemini transcribe models (`gemini-*`) take a separate path; every Chirp / Speech-to-Text model is unchanged.
+      #
+      # @return [OmniAI::Transcribe::Transcription]
+      def self.process!(io, model:, **options)
+        return Gemini.process!(io, model:, **options) if Gemini.model?(model)
+
+        unsupported = options.keys & GEMINI_OPTIONS
+        if unsupported.any?
+          raise ArgumentError, "#{unsupported.join(', ')} only supported by Gemini transcribe models, not #{model}"
+        end
+
+        super
+      end
+
       # @raise [HTTPError]
       #
       # @return [OmniAI::Transcribe::Transcription]

data/lib/omniai/google/version.rb

@@ -2,6 +2,6 @@
 
 module OmniAI
   module Google
-    VERSION = "3.19.0"
+    VERSION = "3.21.0"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: omniai-google
 version: !ruby/object:Gem::Version
-  version: 3.19.0
+  version: 3.21.0
 platform: ruby
 authors:
 - Kevin Sylvestre
@@ -79,6 +79,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: websocket-driver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -123,8 +137,12 @@ files:
 - lib/omniai/google/credentials.rb
 - lib/omniai/google/embed.rb
 - lib/omniai/google/incomplete_stream_error.rb
+- lib/omniai/google/live.rb
+- lib/omniai/google/live/connection.rb
+- lib/omniai/google/live/event.rb
 - lib/omniai/google/stream_error.rb
 - lib/omniai/google/transcribe.rb
+- lib/omniai/google/transcribe/gemini.rb
 - lib/omniai/google/transcribe_helpers.rb
 - lib/omniai/google/upload.rb
 - lib/omniai/google/upload/file.rb