durable_streams 0.1.0

data/lib/durable_streams/stream.rb

@@ -0,0 +1,445 @@
+# frozen_string_literal: true
+
+require "json"
+
+module DurableStreams
+  # Stream handle for read/write operations on a durable stream.
+  class Stream
+    attr_reader :url, :content_type
+
+    # @param url [String] Stream URL or path (resolved against context base_url)
+    # @param context [Context] Configuration context
+    # @param content_type [String, nil] Content type for the stream
+    # @param headers [Hash] Additional headers (merged with context defaults)
+    # @param batching [Boolean] Enable write batching (default: true)
+    def initialize(url, context: DurableStreams.default_context, content_type: nil, headers: {}, batching: true)
+      @url = context.resolve_url(url)
+      @context = context
+      @content_type = content_type
+      @instance_headers = headers || {}
+      @batching = batching
+      # Use mock transport if testing is installed, otherwise create real transport
+      @transport = if defined?(DurableStreams::Testing) && DurableStreams::Testing.transport_if_installed
+                     DurableStreams::Testing.transport_if_installed
+                   else
+                     HTTP::Transport.new(
+                       retry_policy: context.config.retry_policy,
+                       timeout: context.config.timeout
+                     )
+                   end
+      @batch_mutex = Mutex.new
+      @batch_cv = ConditionVariable.new
+      @batch_queue = []
+      @batch_in_flight = false
+    end
+
+    # --- Factory Methods ---
+
+    # Create and verify stream exists
+    # @param url [String] Stream URL or path
+    # @param context [Context] Configuration context
+    # @param headers [Hash] Additional headers
+    # @return [Stream]
+    def self.connect(url, context: DurableStreams.default_context, headers: {}, **options)
+      stream = new(url, context: context, headers: headers, **options)
+      stream.head
+      stream
+    end
+
+    # Create new stream on server
+    # @param url [String] Stream URL or path
+    # @param content_type [Symbol, String] Content type (:json, :bytes, or MIME type)
+    # @param context [Context] Configuration context
+    # @param headers [Hash] Additional headers
+    # @param closed [Boolean] Create stream as immediately closed (default: false)
+    # @return [Stream]
+    def self.create(url, content_type:, context: DurableStreams.default_context, headers: {}, ttl_seconds: nil,
+                    expires_at: nil, body: nil, closed: false, **options)
+      ct = normalize_content_type(content_type)
+      stream = new(url, context: context, content_type: ct, headers: headers, **options)
+      stream.create_stream(content_type: ct, ttl_seconds: ttl_seconds, expires_at: expires_at, body: body, closed: closed)
+      stream
+    end
+
+    # Check if a stream exists without raising
+    # @param url [String] Stream URL or path
+    # @param context [Context] Configuration context
+    # @param headers [Hash] Additional headers
+    # @return [Boolean]
+    def self.exists?(url, context: DurableStreams.default_context, headers: {}, **options)
+      stream = new(url, context: context, headers: headers, **options)
+      stream.exists?
+    end
+
+    # Normalize content type symbol to MIME type
+    def self.normalize_content_type(ct)
+      case ct
+      when :json then "application/json"
+      when :bytes then "application/octet-stream"
+      else ct.to_s
+      end
+    end
+
+    # --- Metadata Operations ---
+
+    # HEAD - Get stream metadata
+    # @return [HeadResult]
+    def head
+      headers = resolved_headers
+      request_url = @url
+
+      response = @transport.request(:head, request_url, headers: headers)
+
+      if response.status == 404
+        raise StreamNotFoundError.new(url: @url)
+      end
+
+      unless response.success?
+        raise DurableStreams.error_from_status(response.status, url: @url, headers: response.headers)
+      end
+
+      @content_type = response["content-type"] if response["content-type"]
+
+      HeadResult.new(
+        exists: true,
+        content_type: response["content-type"],
+        next_offset: response[STREAM_NEXT_OFFSET_HEADER],
+        etag: response["etag"],
+        cache_control: response["cache-control"],
+        stream_closed: response[STREAM_CLOSED_HEADER]&.downcase == "true"
+      )
+    end
+
+    # Check if stream exists without raising
+    # @return [Boolean]
+    def exists?
+      head
+      true
+    rescue StreamNotFoundError
+      false
+    end
+
+    # Check if this is a JSON stream
+    # @return [Boolean]
+    def json?
+      head if @content_type.nil?
+      DurableStreams.json_content_type?(@content_type)
+    end
+
+    # Create stream on server (PUT)
+    # @param content_type [String, nil] Content type for the stream
+    # @param ttl_seconds [Integer, nil] Time-to-live in seconds
+    # @param expires_at [String, nil] Absolute expiry time (RFC3339)
+    # @param body [String, nil] Optional initial body
+    # @param closed [Boolean] Create stream as immediately closed (default: false)
+    def create_stream(content_type: nil, ttl_seconds: nil, expires_at: nil, body: nil, closed: false)
+      headers = resolved_headers
+
+      ct = content_type || @content_type
+      headers["content-type"] = ct if ct
+      headers[STREAM_TTL_HEADER] = ttl_seconds.to_s if ttl_seconds
+      headers[STREAM_EXPIRES_AT_HEADER] = expires_at if expires_at
+      headers[STREAM_CLOSED_HEADER] = "true" if closed
+
+      body_to_send = if body && DurableStreams.json_content_type?(ct)
+                       "[#{body}]"
+                     else
+                       body
+                     end
+
+      response = @transport.request(:put, @url, headers: headers, body: body_to_send)
+
+      if response.status == 409
+        raise StreamExistsError.new(url: @url)
+      end
+
+      unless response.success?
+        raise DurableStreams.error_from_status(response.status, url: @url, body: response.body,
+                                               headers: response.headers)
+      end
+
+      @content_type = response["content-type"] || ct
+    end
+
+    # Delete stream (DELETE)
+    def delete
+      headers = resolved_headers
+
+      response = @transport.request(:delete, @url, headers: headers)
+
+      if response.status == 404
+        raise StreamNotFoundError.new(url: @url)
+      end
+
+      return if response.success? || response.status == 204
+
+      raise DurableStreams.error_from_status(response.status, url: @url, headers: response.headers)
+    end
+
+    # Close the stream (no more appends allowed)
+    # @param data [String, nil] Optional final data to append before closing
+    # @param content_type [String, nil] Content type for the final data
+    # @return [CloseResult]
+    def close_stream(data: nil, content_type: nil)
+      headers = resolved_headers
+      headers[STREAM_CLOSED_HEADER] = "true"
+
+      ct = content_type || @content_type
+      headers["content-type"] = ct if ct
+
+      # For JSON streams, wrap data in array if needed
+      body = if data && DurableStreams.json_content_type?(ct)
+               "[#{data}]"
+             else
+               data
+             end
+
+      response = @transport.request(:post, @url, headers: headers, body: body)
+
+      # 204 means idempotent close (already closed)
+      if response.status == 204
+        next_offset = response[STREAM_NEXT_OFFSET_HEADER] || "-1"
+        return CloseResult.new(final_offset: next_offset)
+      end
+
+      # 409 with Stream-Closed header means already closed - treat as idempotent success
+      if response.status == 409 && response[STREAM_CLOSED_HEADER]&.downcase == "true"
+        next_offset = response[STREAM_NEXT_OFFSET_HEADER] || "-1"
+        return CloseResult.new(final_offset: next_offset)
+      end
+
+      unless response.success?
+        raise DurableStreams.error_from_status(response.status, url: @url, body: response.body,
+                                               headers: response.headers)
+      end
+
+      next_offset = response[STREAM_NEXT_OFFSET_HEADER]
+      raise FetchError.new("Server did not return #{STREAM_NEXT_OFFSET_HEADER} header", url: @url) unless next_offset
+
+      CloseResult.new(final_offset: next_offset)
+    end
+
+    # --- Write Operations ---
+
+    # Append data to stream
+    # For JSON streams, pass pre-serialized JSON strings.
+    # @param data [String] Data to append (pre-serialized JSON for JSON streams)
+    # @param seq [String, nil] Optional sequence number for ordering
+    # @return [AppendResult]
+    # @example
+    #   # JSON stream - pass pre-serialized JSON
+    #   stream.append(JSON.generate({ message: "hello" }))
+    #
+    #   # Byte stream
+    #   stream.append("raw text data")
+    def append(data, seq: nil)
+      unless data.is_a?(String)
+        raise ArgumentError, "append() requires a String. For objects, use JSON.generate(). Got #{data.class}"
+      end
+
+      if @batching
+        append_with_batching(data, seq)
+      else
+        append_direct(data, seq)
+      end
+    end
+
+    # Sync append (same as append, explicit name for clarity)
+    # @param data [String] Data to append (pre-serialized JSON for JSON streams)
+    # @param seq [String, nil] Optional sequence number
+    # @return [AppendResult]
+    def append!(data, seq: nil)
+      append(data, seq: seq)
+    end
+
+    # Shovel operator for append
+    # @param data [String] Data to append (pre-serialized JSON for JSON streams)
+    # @return [self] Returns self for chaining
+    def <<(data)
+      append(data)
+      self
+    end
+
+    # --- Read Operations ---
+
+    # Read from stream
+    # @param offset [String] Starting offset (default: "-1" for beginning)
+    # @param live [Boolean, Symbol] Live mode (false, :long_poll, :sse)
+    # @param format [Symbol] Format hint (:auto, :json, :bytes)
+    # @param cursor [String, nil] Optional cursor for continuation
+    # @yield [Reader] Optional block for automatic cleanup
+    # @return [JsonReader, ByteReader] Reader for iterating messages
+    def read(offset: "-1", live: false, format: :auto, cursor: nil, &block)
+      reader = create_reader(offset: offset, live: live, format: format, cursor: cursor)
+
+      if block_given?
+        begin
+          yield reader
+        ensure
+          reader.close
+        end
+      else
+        reader
+      end
+    end
+
+    # Iterate over messages (catch-up only)
+    # @yield [Object] Each message
+    # @return [Enumerator] If no block given
+    def each(&block)
+      return enum_for(:each) unless block_given?
+
+      read(live: false).each(&block)
+    end
+
+    # Convenience: Read all current data
+    # @param offset [String] Starting offset
+    # @return [Array] All messages from offset to current end
+    def read_all(offset: "-1")
+      read(offset: offset, live: false, &:to_a)
+    end
+
+    # Shutdown the transport
+    def close
+      @transport.shutdown
+    end
+
+    # --- Internal Accessors ---
+
+    attr_reader :transport, :context
+
+    # Resolve headers for requests (used by readers)
+    # @param extra [Hash] Additional headers to merge
+    # @return [Hash] Resolved headers
+    def resolved_headers(extra = {})
+      base = HTTP.resolve_headers(@context.config.default_headers)
+      base.merge(HTTP.resolve_headers(@instance_headers)).merge(extra)
+    end
+
+    private
+
+    def create_reader(offset:, live:, format:, cursor:)
+      effective_format = determine_format(format)
+
+      case effective_format
+      when :json
+        JsonReader.new(self, offset: offset, live: live, cursor: cursor)
+      else
+        ByteReader.new(self, offset: offset, live: live, cursor: cursor)
+      end
+    end
+
+    def determine_format(format)
+      return format if format != :auto
+
+      head if @content_type.nil?
+      DurableStreams.json_content_type?(@content_type) ? :json : :bytes
+    end
+
+    def append_direct(data, seq)
+      post_append([data], seq: seq)
+    end
+
+    def append_with_batching(data, seq)
+      queue_entry = { data: data, seq: seq, result: nil, error: nil, done: false }
+      is_leader = false
+
+      @batch_mutex.synchronize do
+        @batch_queue << queue_entry
+        unless @batch_in_flight
+          @batch_in_flight = true
+          is_leader = true
+        end
+      end
+
+      flush_batch if is_leader
+
+      @batch_mutex.synchronize do
+        @batch_cv.wait(@batch_mutex) until queue_entry[:done]
+      end
+
+      raise queue_entry[:error] if queue_entry[:error]
+
+      queue_entry[:result]
+    end
+
+    def flush_batch
+      loop do
+        messages = nil
+        @batch_mutex.synchronize do
+          if @batch_queue.empty?
+            @batch_in_flight = false
+            return
+          end
+          messages = @batch_queue.dup
+          @batch_queue.clear
+        end
+
+        begin
+          result = send_batch(messages)
+          @batch_mutex.synchronize do
+            messages.each do |msg|
+              msg[:result] = result
+              msg[:done] = true
+            end
+            @batch_cv.broadcast
+          end
+        rescue StandardError => e
+          @batch_mutex.synchronize do
+            messages.each do |msg|
+              msg[:error] = e
+              msg[:done] = true
+            end
+            @batch_queue.each do |msg|
+              msg[:error] = e
+              msg[:done] = true
+            end
+            @batch_queue.clear
+            @batch_in_flight = false
+            @batch_cv.broadcast
+          end
+          return
+        end
+      end
+    end
+
+    def send_batch(messages)
+      highest_seq = messages.reverse.find { |m| m[:seq] }&.fetch(:seq)
+      post_append(messages.map { |m| m[:data] }, seq: highest_seq)
+    end
+
+    def post_append(data_items, seq: nil)
+      headers = resolved_headers
+      headers["content-type"] = @content_type if @content_type
+      headers[STREAM_SEQ_HEADER] = seq.to_s if seq
+
+      # data_items are pre-serialized strings
+      body = if DurableStreams.json_content_type?(@content_type)
+               # Wrap pre-serialized JSON strings in array
+               "[#{data_items.join(',')}]"
+             else
+               data_items.join
+             end
+
+      response = @transport.request(:post, @url, headers: headers, body: body)
+
+      if response.status == 409
+        if response[STREAM_CLOSED_HEADER]&.downcase == "true"
+          raise StreamClosedError.new(url: @url)
+        end
+
+        raise SeqConflictError.new(url: @url)
+      end
+
+      unless response.success? || response.status == 204
+        raise DurableStreams.error_from_status(response.status, url: @url, body: response.body,
+                                               headers: response.headers)
+      end
+
+      next_offset = response[STREAM_NEXT_OFFSET_HEADER]
+      raise FetchError.new("Server did not return #{STREAM_NEXT_OFFSET_HEADER} header", url: @url) unless next_offset
+
+      AppendResult.new(next_offset: next_offset)
+    end
+  end
+end

data/lib/durable_streams/testing.rb

@@ -0,0 +1,277 @@
+# frozen_string_literal: true
+
+module DurableStreams
+  # Testing utilities for DurableStreams.
+  # Provides mock transport and matchers for testing without a real server.
+  module Testing
+    # Mock response for testing
+    class MockResponse
+      attr_reader :status, :body, :headers
+
+      def initialize(status:, body: "", headers: {})
+        @status = status
+        @body = body
+        @headers = headers.transform_keys(&:downcase)
+      end
+
+      def success?
+        status >= 200 && status < 300
+      end
+
+      def [](header)
+        @headers[header.to_s.downcase]
+      end
+    end
+
+    # Mock transport for testing without a real server
+    class MockTransport
+      attr_reader :requests, :streams
+
+      def initialize
+        @requests = []
+        @streams = {}
+        @responses = {}
+        @default_offset = "0"
+      end
+
+      # Record a request and return a mock response
+      def request(method, url, headers: {}, body: nil, **_options)
+        uri = URI.parse(url)
+        path = uri.path
+
+        @requests << {
+          method: method,
+          url: url,
+          path: path,
+          headers: headers,
+          body: body
+        }
+
+        # Return configured response or generate a default
+        response_key = "#{method}:#{path}"
+        if @responses[response_key]
+          @responses[response_key]
+        else
+          default_response(method, path, body)
+        end
+      end
+
+      # Stream request (simplified mock - just yields response)
+      def stream_request(method, url, headers: {}, timeout: nil)
+        response = request(method, url, headers: headers)
+        yield response if block_given?
+      end
+
+      # Configure a response for a specific method and path
+      # @param method [Symbol] HTTP method
+      # @param path [String] URL path
+      # @param status [Integer] Response status code
+      # @param body [String] Response body
+      # @param headers [Hash] Response headers
+      def on(method, path, status:, body: "", headers: {})
+        @responses["#{method}:#{path}"] = MockResponse.new(
+          status: status,
+          body: body,
+          headers: headers
+        )
+      end
+
+      # Add messages to a mock stream (for read testing)
+      # @param path [String] Stream path
+      # @param messages [Array] Messages to add
+      def seed_stream(path, messages)
+        @streams[path] ||= []
+        @streams[path].concat(messages)
+      end
+
+      # Get messages for a stream
+      # @param path [String] Stream path
+      # @return [Array] Messages
+      def messages_for(path)
+        @streams[path] || []
+      end
+
+      # Clear all recorded requests and mock data
+      def clear!
+        @requests.clear
+        @streams.clear
+        @responses.clear
+      end
+
+      # No-op shutdown for mock
+      def shutdown; end
+
+      private
+
+      def default_response(method, path, body)
+        case method
+        when :put
+          # Stream creation
+          @streams[path] ||= []
+          MockResponse.new(
+            status: 201,
+            headers: { "content-type" => "application/json" }
+          )
+        when :head
+          if @streams.key?(path)
+            MockResponse.new(
+              status: 200,
+              headers: {
+                "content-type" => "application/json",
+                STREAM_NEXT_OFFSET_HEADER => @default_offset
+              }
+            )
+          else
+            MockResponse.new(status: 404)
+          end
+        when :post
+          # Append
+          @streams[path] ||= []
+          if body
+            begin
+              messages = JSON.parse(body)
+              @streams[path].concat(Array(messages))
+            rescue JSON::ParserError
+              @streams[path] << body
+            end
+          end
+          new_offset = @streams[path].size.to_s
+          MockResponse.new(
+            status: 200,
+            headers: { STREAM_NEXT_OFFSET_HEADER => new_offset }
+          )
+        when :get
+          messages = @streams[path] || []
+          MockResponse.new(
+            status: 200,
+            body: JSON.generate(messages),
+            headers: {
+              "content-type" => "application/json",
+              STREAM_NEXT_OFFSET_HEADER => messages.size.to_s,
+              STREAM_UP_TO_DATE_HEADER => "true"
+            }
+          )
+        when :delete
+          if @streams.delete(path)
+            MockResponse.new(status: 204)
+          else
+            MockResponse.new(status: 404)
+          end
+        else
+          MockResponse.new(status: 405)
+        end
+      end
+    end
+
+    class << self
+      # Get the shared mock transport instance
+      # @return [MockTransport]
+      def mock_transport
+        @mock_transport ||= MockTransport.new
+      end
+
+      # Install mock transport (replaces transport in new Streams)
+      def install!
+        @installed = true
+        DurableStreams.reset_configuration!
+      end
+
+      # Check if testing mode is installed
+      def installed?
+        @installed || false
+      end
+
+      # Get transport for a stream (mock if installed, nil otherwise)
+      # Called by Stream to check if mock transport should be used
+      # @return [MockTransport, nil]
+      def transport_if_installed
+        installed? ? mock_transport : nil
+      end
+
+      # Reset testing state
+      def reset!
+        @mock_transport = nil
+        @installed = false
+        DurableStreams.reset_configuration!
+      end
+
+      # Get messages appended to a stream path
+      # @param path [String] Stream path
+      # @return [Array] Messages
+      def messages_for(path)
+        mock_transport.messages_for(path)
+      end
+
+      # Get all recorded requests
+      # @return [Array<Hash>] Requests
+      def requests
+        mock_transport.requests
+      end
+
+      # Clear all test state
+      def clear!
+        mock_transport.clear!
+      end
+    end
+
+    # RSpec matchers (only defined if RSpec is available)
+    if defined?(RSpec)
+      require "rspec/expectations"
+
+      RSpec::Matchers.define :have_appended_to do |path|
+        match do |_|
+          messages = DurableStreams::Testing.messages_for(path)
+          if @expected_data
+            messages.any? { |m| hash_subset?(m, @expected_data) }
+          else
+            messages.any?
+          end
+        end
+
+        chain :with do |data|
+          @expected_data = data
+        end
+
+        failure_message do
+          messages = DurableStreams::Testing.messages_for(path)
+          if @expected_data
+            "expected to find message matching #{@expected_data.inspect} in #{messages.inspect}"
+          else
+            "expected messages to be appended to #{path}, but none were"
+          end
+        end
+
+        def hash_subset?(actual, expected)
+          return actual == expected unless expected.is_a?(Hash) && actual.is_a?(Hash)
+
+          expected.all? { |k, v| actual.key?(k.to_s) && hash_subset?(actual[k.to_s], v) }
+        end
+      end
+
+      RSpec::Matchers.define :have_made_request do |method|
+        match do |_|
+          requests = DurableStreams::Testing.requests
+          requests.any? do |req|
+            matches_method = req[:method] == method
+            matches_path = @to_path.nil? || req[:path] == @to_path
+            matches_body = @with_body.nil? || req[:body]&.include?(@with_body.to_s)
+            matches_method && matches_path && matches_body
+          end
+        end
+
+        chain :to do |path|
+          @to_path = path
+        end
+
+        chain :with_body do |body|
+          @with_body = body
+        end
+
+        failure_message do
+          requests = DurableStreams::Testing.requests
+          "expected #{method} request#{@to_path ? " to #{@to_path}" : ""}, but got: #{requests.map { |r| "#{r[:method]} #{r[:path]}" }}"
+        end
+      end
+    end
+  end
+end