durable_streams 0.1.0

data/lib/durable_streams/producer.rb

@@ -0,0 +1,436 @@
+# frozen_string_literal: true
+
+require "json"
+
+module DurableStreams
+  # Producer for exactly-once writes with batching.
+  # Uses producer_id, epoch, and sequence numbers to ensure exactly-once delivery.
+  class Producer
+    attr_reader :epoch, :seq
+
+    # Open a producer with block form for automatic cleanup
+    # @example
+    #   Producer.open(url: "...", producer_id: "...") do |producer|
+    #     producer << data
+    #   end # auto flush/close
+    # @yield [Producer] The producer instance
+    # @return [Object] The block's return value
+    def self.open(**options, &block)
+      producer = new(**options)
+      return producer unless block_given?
+
+      begin
+        yield producer
+      ensure
+        # Preserve original exception if close also raises
+        begin
+          producer.close
+        rescue StandardError => close_error
+          raise unless $!  # Re-raise close error if no original exception
+
+          DurableStreams.logger&.warn(
+            "Error during producer close (original exception preserved): #{close_error.class}: #{close_error.message}"
+          )
+        end
+      end
+    end
+
+    # @param url [String] Stream URL
+    # @param producer_id [String] Stable identifier for this producer
+    # @param epoch [Integer] Starting epoch (increment on restart)
+    # @param auto_claim [Boolean] Auto-retry with epoch+1 on 403
+    # @param max_batch_bytes [Integer] Max bytes before flush (default: 1MB)
+    # @param linger_ms [Integer] Max wait before flush (default: 5ms)
+    # @param max_in_flight [Integer] Max concurrent batches (default: 5)
+    # @param content_type [String] Content type for the stream
+    # @param headers [Hash] Additional headers
+    def initialize(url:, producer_id:, epoch: 0, auto_claim: false,
+                   max_batch_bytes: 1_048_576, linger_ms: 5, max_in_flight: 5,
+                   content_type: nil, headers: {}, next_seq: 0)
+      @url = url
+      @producer_id = producer_id
+      @epoch = epoch
+      @auto_claim = auto_claim
+      @max_batch_bytes = max_batch_bytes
+      @linger_ms = linger_ms
+      @max_in_flight = max_in_flight
+      @content_type = content_type || "application/json"
+      @headers = headers
+
+      raise ArgumentError, "next_seq must be >= 0" if next_seq.negative?
+
+      @seq = next_seq - 1 # Start at next_seq - 1 so next append uses next_seq
+      @pending = []
+      @mutex = Mutex.new
+      @send_mutex = Mutex.new  # Ensure batches are sent in order
+      @in_flight = 0
+      @in_flight_cv = ConditionVariable.new
+      @transport = HTTP::Transport.new
+      @closed = false
+      @stream_closed = false
+      @linger_timer = nil
+      @linger_cancelled = false
+      @batch_queue = Queue.new
+      @sender_thread = nil
+      @last_error = nil
+    end
+
+    # Append a message (fire-and-forget, batched)
+    # For JSON streams, pass pre-serialized JSON strings.
+    # @param data [String] Data to append (pre-serialized JSON for JSON streams)
+    # @example
+    #   producer.append(JSON.generate({ message: "hello" }))
+    def append(data)
+      raise ClosedError.new("Producer is closed", url: @url) if @closed
+      unless data.is_a?(String)
+        raise ArgumentError, "append() requires a String. For objects, use JSON.generate(). Got #{data.class}"
+      end
+
+      batch_to_send = nil
+      @mutex.synchronize do
+        @seq += 1
+        @pending << { data: data, seq: @seq }
+
+        # Start linger timer if this is first message in batch
+        start_linger_timer if @pending.size == 1 && @linger_ms > 0
+
+        # Flush if batch is full
+        if batch_size_bytes >= @max_batch_bytes
+          batch_to_send = @pending.dup
+          @pending.clear
+          cancel_linger_timer
+        end
+      end
+
+      # Send outside the mutex to avoid blocking other appends
+      queue_batch(batch_to_send) if batch_to_send
+    end
+
+    # Shovel operator for append (Ruby idiom)
+    # @param data [String] Data to append (pre-serialized JSON for JSON streams)
+    # @return [self] Returns self for chaining
+    def <<(data)
+      append(data)
+      self
+    end
+
+    # Append and wait for acknowledgment (sync/blocking)
+    # @param data [String] Data to append (pre-serialized JSON for JSON streams)
+    # @return [ProducerResult]
+    def append!(data)
+      append(data)
+      flush
+      ProducerResult.new(
+        next_offset: nil, # We don't track individual message offsets in batched mode
+        duplicate: false,
+        epoch: @epoch,
+        seq: @seq
+      )
+    end
+
+    # Flush all pending batches
+    def flush
+      batch = nil
+      @mutex.synchronize do
+        cancel_linger_timer
+        # Check for errors from background threads
+        raise @last_error if @last_error
+
+        return if @pending.empty?
+
+        batch = @pending.dup
+        @pending.clear
+      end
+
+      # Send synchronously for flush
+      send_batch_sync(batch) if batch && !batch.empty?
+
+      # Wait for all in-flight batches to complete
+      wait_for_inflight
+
+      # Check for errors that occurred during wait
+      @mutex.synchronize { raise @last_error if @last_error }
+    end
+
+    # Close the producer, flushing pending data
+    def close
+      return if @closed
+
+      @closed = true
+      cancel_linger_timer
+      flush
+
+      # Signal sender thread to stop and wait for it
+      if @sender_thread&.alive?
+        @batch_queue << :shutdown
+        @sender_thread.join(5) # Wait up to 5 seconds
+        @sender_thread.kill if @sender_thread.alive? # Force kill if stuck
+      end
+    end
+
+    # Close the stream using producer headers (idempotent)
+    # @param data [String, nil] Optional final data to append before closing
+    def close_stream(data: nil)
+      return if @stream_closed
+
+      flush
+
+      attempts = 0
+      close_seq = @seq + 1
+
+      begin
+        send_close_request(data, close_seq, @epoch)
+        @seq = close_seq
+        @stream_closed = true
+        @closed = true
+      rescue StaleEpochError => e
+        attempts += 1
+        raise if attempts > 3
+
+        if @auto_claim
+          server_epoch = e.current_epoch || @epoch
+          @epoch = [server_epoch + 1, @epoch + 1].max
+          close_seq = 0
+          retry
+        else
+          raise
+        end
+      end
+    end
+
+    # Check if the producer has been closed
+    # @return [Boolean]
+    def closed?
+      @closed
+    end
+
+    private
+
+    def batch_size_bytes
+      # Data is now pre-serialized strings
+      @pending.sum { |msg| msg[:data].bytesize }
+    end
+
+    def start_linger_timer
+      return if @linger_ms <= 0
+
+      @linger_cancelled = false
+      @linger_timer = Thread.new do
+        begin
+          sleep(@linger_ms / 1000.0)
+          flush unless @closed || @linger_cancelled
+        rescue StandardError => e
+          @mutex.synchronize do
+            if @last_error
+              DurableStreams.logger&.warn(
+                "Additional error in linger timer (first error already recorded): #{e.class}: #{e.message}"
+              )
+            else
+              @last_error = e
+            end
+          end
+        end
+      end
+    end
+
+    def cancel_linger_timer
+      @linger_cancelled = true
+      @linger_timer&.kill
+      @linger_timer = nil
+    end
+
+    def queue_batch(batch)
+      return if batch.nil? || batch.empty?
+
+      # If max_in_flight is 1 or linger_ms is 0, send synchronously for ordering
+      if @max_in_flight <= 1 || @linger_ms == 0
+        send_batch_sync(batch)
+      else
+        # For batched mode with concurrency, use the queue
+        start_sender_thread
+        @batch_queue << batch
+      end
+    end
+
+    def start_sender_thread
+      return if @sender_thread&.alive?
+
+      @sender_thread = Thread.new do
+        begin
+          loop do
+            batch = @batch_queue.pop
+            break if batch == :shutdown
+
+            send_batch_sync(batch)
+          end
+        rescue StandardError => e
+          DurableStreams.logger&.error(
+            "Sender thread died unexpectedly: #{e.class}: #{e.message}"
+          )
+          @mutex.synchronize do
+            if @last_error
+              DurableStreams.logger&.warn(
+                "Sender thread error (first error already recorded): #{e.class}: #{e.message}"
+              )
+            else
+              @last_error = e
+            end
+          end
+        end
+      end
+    end
+
+    def wait_for_inflight
+      @mutex.synchronize do
+        while @in_flight > 0
+          @in_flight_cv.wait(@mutex, 0.1)
+        end
+      end
+    end
+
+    def send_batch_sync(batch, retry_count: 0)
+      return if batch.empty?
+
+      # Serialize batch sending to ensure sequence order
+      @send_mutex.synchronize do
+        # Wait for in-flight slot
+        @mutex.synchronize do
+          while @in_flight >= @max_in_flight
+            @in_flight_cv.wait(@mutex, 0.1)
+          end
+          @in_flight += 1
+        end
+
+        begin
+          send_batch_request(batch)
+        rescue StaleEpochError => e
+          if @auto_claim && retry_count < 3
+            new_epoch = nil
+            new_batch = nil
+            @mutex.synchronize do
+              # Use the server's current epoch + 1, or at minimum our epoch + 1
+              server_epoch = e.current_epoch || @epoch
+              new_epoch = [server_epoch + 1, @epoch + 1].max
+              @epoch = new_epoch
+              # Rebuild the batch with seq starting from 0 for the new epoch
+              new_batch = batch.each_with_index.map do |msg, idx|
+                { data: msg[:data], seq: idx }
+              end
+              # Update @seq to the last seq in the batch so subsequent appends continue correctly
+              # Any pending messages will be re-sequenced on next flush
+              @seq = new_batch.size - 1
+              # Re-sequence any pending messages to continue after the batch
+              @pending.each_with_index do |msg, idx|
+                msg[:seq] = @seq + 1 + idx
+              end
+              @seq += @pending.size
+            end
+            send_batch_request_with_epoch(new_batch, new_epoch)
+          else
+            raise
+          end
+        ensure
+          @mutex.synchronize do
+            @in_flight -= 1
+            @in_flight_cv.broadcast
+          end
+        end
+      end
+    end
+
+    def send_batch_request(batch)
+      send_batch_request_with_epoch(batch, @epoch)
+    end
+
+    def send_batch_request_with_epoch(batch, epoch)
+      headers = HTTP.resolve_headers(@headers)
+      headers["content-type"] = @content_type
+      headers[PRODUCER_ID_HEADER] = @producer_id
+      headers[PRODUCER_EPOCH_HEADER] = epoch.to_s
+
+      # Use the first message's seq as the starting seq
+      first_seq = batch.first[:seq]
+      headers[PRODUCER_SEQ_HEADER] = first_seq.to_s
+
+      # Build body - data is pre-serialized strings
+      body = if DurableStreams.json_content_type?(@content_type)
+               # Wrap pre-serialized JSON strings in array
+               "[#{batch.map { |m| m[:data] }.join(',')}]"
+             else
+               batch.map { |m| m[:data] }.join
+             end
+
+      response = @transport.request(:post, @url, headers: headers, body: body)
+
+      case response.status
+      when 200, 201, 204
+        # Success
+        nil
+      when 403
+        # Stale epoch
+        current_epoch = response[PRODUCER_EPOCH_HEADER]&.to_i
+        raise StaleEpochError.new(current_epoch: current_epoch, url: @url, headers: response.headers)
+      when 409
+        if response[STREAM_CLOSED_HEADER]&.downcase == "true"
+          raise StreamClosedError.new(url: @url, headers: response.headers)
+        end
+
+        # Could be sequence gap or other conflict
+        expected = response[PRODUCER_EXPECTED_SEQ_HEADER]&.to_i
+        received = response[PRODUCER_RECEIVED_SEQ_HEADER]&.to_i
+        if expected && received
+          raise SequenceGapError.new(expected_seq: expected, received_seq: received,
+                                     url: @url, headers: response.headers)
+        else
+          raise SeqConflictError.new(url: @url, headers: response.headers)
+        end
+      else
+        raise DurableStreams.error_from_status(response.status, url: @url, body: response.body,
+                                               headers: response.headers)
+      end
+    end
+
+    def send_close_request(data, seq, epoch)
+      headers = HTTP.resolve_headers(@headers)
+      headers["content-type"] = @content_type
+      headers[PRODUCER_ID_HEADER] = @producer_id
+      headers[PRODUCER_EPOCH_HEADER] = epoch.to_s
+      headers[PRODUCER_SEQ_HEADER] = seq.to_s
+      headers[STREAM_CLOSED_HEADER] = "true"
+
+      body = if data.nil?
+               ""
+             elsif DurableStreams.json_content_type?(@content_type)
+               "[#{data}]"
+             else
+               data
+             end
+
+      response = @transport.request(:post, @url, headers: headers, body: body)
+
+      case response.status
+      when 200, 201, 204
+        nil
+      when 403
+        current_epoch = response[PRODUCER_EPOCH_HEADER]&.to_i
+        raise StaleEpochError.new(current_epoch: current_epoch, url: @url, headers: response.headers)
+      when 409
+        if response[STREAM_CLOSED_HEADER]&.downcase == "true"
+          raise StreamClosedError.new(url: @url, headers: response.headers)
+        end
+
+        expected = response[PRODUCER_EXPECTED_SEQ_HEADER]&.to_i
+        received = response[PRODUCER_RECEIVED_SEQ_HEADER]&.to_i
+        if expected && received
+          raise SequenceGapError.new(expected_seq: expected, received_seq: received,
+                                     url: @url, headers: response.headers)
+        else
+          raise SeqConflictError.new(url: @url, headers: response.headers)
+        end
+      else
+        raise DurableStreams.error_from_status(response.status, url: @url, body: response.body,
+                                               headers: response.headers)
+      end
+    end
+  end
+end

data/lib/durable_streams/sse_reader.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+require "json"
+require "uri"
+require "net/http"
+require "base64"
+
+module DurableStreams
+  # SSE (Server-Sent Events) reader for live streaming
+  class SSEReader
+    attr_reader :next_offset, :cursor, :up_to_date, :status
+
+    # @param stream [Stream] Parent stream handle
+    # @param offset [String] Starting offset
+    # @param cursor [String, nil] Initial cursor
+    # @param retry_policy [RetryPolicy, nil] Retry policy for reconnection
+    def initialize(stream, offset: "-1", cursor: nil, retry_policy: nil)
+      @stream = stream
+      @offset = offset
+      @next_offset = offset
+      @cursor = cursor
+      @retry_policy = retry_policy || RetryPolicy.default
+      @up_to_date = false
+      @closed = false
+      @status = nil
+      @buffer = +""
+      @http_response = nil
+      @connection = nil
+      @encoding = nil
+    end
+
+    # Iterate over SSE events
+    # @yield [Hash] Event with :type, :data, :next_offset, :cursor, :up_to_date
+    def each_event(&block)
+      return enum_for(:each_event) unless block_given?
+
+      with_reconnection do
+        open_sse_connection do |response|
+          @http_response = response
+
+          response.read_body do |chunk|
+            break if @closed
+
+            @buffer << chunk
+            parse_events.each do |event|
+              yield event
+              break if @closed
+            end
+          end
+        end
+      end
+    end
+
+    # Close the SSE connection
+    def close
+      @closed = true
+      begin
+        @http_response&.instance_variable_get(:@socket)&.close
+      rescue StandardError => e
+        DurableStreams.logger&.warn("SSE socket close error (expected during cleanup): #{e.class}: #{e.message}")
+      end
+      begin
+        @connection&.finish
+      rescue StandardError => e
+        DurableStreams.logger&.warn("SSE connection finish error (expected during cleanup): #{e.class}: #{e.message}")
+      end
+    end
+
+    def closed?
+      @closed
+    end
+
+    private
+
+    def with_reconnection
+      attempts = 0
+      last_error = nil
+      begin
+        yield
+      rescue IOError, Errno::ECONNRESET, Net::ReadTimeout, Errno::EPIPE => e
+        return if @closed
+
+        last_error = e
+        attempts += 1
+        if attempts > @retry_policy.max_retries
+          raise ConnectionError.new("SSE connection failed after #{attempts} retries: #{e.message}")
+        end
+
+        delay = [@retry_policy.initial_delay * (@retry_policy.multiplier**(attempts - 1)),
+                 @retry_policy.max_delay].min
+        sleep(delay)
+        @buffer = +""
+        retry
+      end
+    end
+
+    def open_sse_connection(&block)
+      params = { offset: @next_offset, live: "sse" }
+      params[:cursor] = @cursor if @cursor
+      request_url = HTTP.build_url(@stream.url, params)
+      uri = URI.parse(request_url)
+
+      @connection = Net::HTTP.new(uri.host, uri.port)
+      @connection.use_ssl = uri.scheme == "https"
+      @connection.open_timeout = 10
+      @connection.read_timeout = 300 # Long timeout for SSE
+      @connection.start
+
+      path = uri.path
+      path = "/" if path.empty?
+      path = "#{path}?#{uri.query}" if uri.query
+
+      request = Net::HTTP::Get.new(path)
+      # Apply user headers first, then force Accept header for SSE
+      @stream.resolved_headers.each { |k, v| request[k] = v }
+      request["Accept"] = "text/event-stream"
+
+      @connection.request(request) do |response|
+        @status = response.code.to_i
+        if @status == 404
+          raise StreamNotFoundError.new(url: @stream.url)
+        end
+        unless @status >= 200 && @status < 300
+          raise DurableStreams.error_from_status(@status, url: @stream.url)
+        end
+        # Detect encoding from response header (server auto-detects binary content types)
+        encoding_header = response["stream-sse-data-encoding"]
+        @encoding = encoding_header if encoding_header && !encoding_header.empty?
+        yield response
+      end
+    ensure
+      begin
+        @connection&.finish
+      rescue StandardError => e
+        DurableStreams.logger&.warn("SSE connection cleanup error: #{e.class}: #{e.message}")
+      end
+    end
+
+    def parse_events
+      events = []
+      # Handle both \n\n and \r\n\r\n delimiters
+      while (match = @buffer.match(/\r?\n\r?\n/))
+        idx = match.begin(0)
+        raw = @buffer.slice!(0, idx + match[0].length)
+        event = parse_sse_event(raw)
+        events << event if event
+      end
+      events
+    end
+
+    def parse_sse_event(raw)
+      event_type = nil
+      data_lines = []
+
+      raw.each_line do |line|
+        line = line.chomp
+        next if line.start_with?(":") # Comment line
+        next if line.empty?
+
+        case line
+        when /^event:\s*(.*)$/
+          event_type = ::Regexp.last_match(1)
+        when /^data:\s?(.*)$/
+          data_lines << ::Regexp.last_match(1)
+        when /^data$/
+          data_lines << "" # Empty data line
+        end
+      end
+
+      return nil if data_lines.empty? && event_type != "control"
+
+      data = data_lines.join("\n")
+
+      # Parse control events for metadata
+      if event_type == "control"
+        # Validate control event data
+        if data.nil? || data.strip.empty?
+          raise ParseError.new("Empty control event data")
+        end
+
+        begin
+          control = JSON.parse(data)
+          # Must be a JSON object
+          unless control.is_a?(Hash)
+            raise ParseError.new("Control event data is not a JSON object")
+          end
+          @next_offset = control["streamNextOffset"] if control["streamNextOffset"]
+          @cursor = control["streamCursor"] if control["streamCursor"]
+          @up_to_date = control["upToDate"] == true || control["streamUpToDate"] == true
+          return {
+            type: "control",
+            data: nil, # No data payload for control events
+            next_offset: @next_offset,
+            cursor: @cursor,
+            up_to_date: @up_to_date
+          }
+        rescue JSON::ParserError => e
+          raise ParseError.new("Malformed control event JSON: #{e.message}")
+        end
+      end
+
+      # Only process known event types: "data", "message", or nil (default)
+      # Ignore unknown event types per SSE spec (forward compatibility)
+      unless event_type.nil? || event_type == "data" || event_type == "message"
+        return nil
+      end
+
+      # Decode base64 if encoding is set (Protocol Section 5.7)
+      # Per protocol: remove \n and \r characters before base64 decoding
+      if @encoding == "base64" && data && !data.empty?
+        cleaned_data = data.gsub(/[\n\r]/, "")
+        begin
+          data = Base64.strict_decode64(cleaned_data)
+        rescue ArgumentError => e
+          raise ParseError.new("Invalid base64 data in SSE event: #{e.message}")
+        end
+      end
+
+      {
+        type: event_type,
+        data: data,
+        next_offset: @next_offset,
+        cursor: @cursor,
+        up_to_date: @up_to_date
+      }
+    end
+  end
+end