loams 0.1.0

data/lib/loams/invoker.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require "loams/codec"
+require "loams/envelopes"
+require "loams/errors"
+require "loams/idempotency"
+require "loams/retry"
+require "loams/token_source"
+require "loams/streams"
+
+module Loams
+  class CallInvoker
+    attr_reader :transport, :protocol, :codec, :token_source, :retry_policy
+
+    def initialize(
+      transport:,
+      protocol: :connect,
+      codec: :proto,
+      token_source: nil,
+      retry_policy: nil
+    )
+      @transport = transport
+      @protocol = protocol
+      @codec = codec
+      @token_source = token_source.is_a?(TokenSource) ? token_source : (token_source ? TokenSource.new(token_source) : nil)
+      @retry_policy = retry_policy || RetryPolicy.new
+    end
+
+    def unary(binding, request, options = {})
+      # Decide idempotency key
+      mint = options.fetch(:mint_idempotency_key, true)
+      key = Idempotency.apply(binding, request, options[:idempotency_key], mint: mint)
+
+      attempt = 0
+      refreshed_token = false
+
+      loop do
+        attempt += 1
+
+        # Build headers
+        ct = ContentTypes.content_type_for(@protocol, @codec, streaming: false)
+        headers = { "content-type" => ct }
+
+        if (tok = current_token)
+          headers["authorization"] = "Bearer #{tok}"
+        end
+
+        if key && !key.empty?
+          headers["idempotency-key"] = key
+        end
+
+        if options[:headers]
+          options[:headers].each { |k, v| headers[k.to_s.downcase] = v.to_s }
+        end
+
+        # Serialize body
+        body = MessageCodec.serialize(binding.request, request, @codec)
+        if @protocol == :grpc_web
+          body = Envelopes.wrap(body, Envelopes::FLAG_DATA)
+        end
+
+        req = Request.new(
+          method: "POST",
+          path: "/#{binding.rpc}",
+          headers: headers,
+          body: body,
+          rpc: binding.rpc,
+        )
+
+        res = begin
+          @transport.call_unary(req)
+        rescue StandardError => e
+          raise ErrorParser.build_error(:unknown, e.message, nil, nil, binding.rpc)
+        end
+
+        # Handle gRPC-Web trailers in headers
+        if @protocol == :grpc_web && (res.headers["grpc-status"] || res.headers["Grpc-Status"])
+          status_num = (res.headers["grpc-status"] || res.headers["Grpc-Status"]).to_i
+          if status_num != 0
+            err = ErrorParser.parse_grpc_trailers(res.headers, res.status, binding.rpc)
+            if can_refresh_and_retry?(err, refreshed_token)
+              refreshed_token = true
+              next
+            end
+            if can_retry?(err, binding, request, options, attempt)
+              sleep(@retry_policy.backoff_duration(attempt))
+              next
+            end
+            raise err
+          end
+        end
+
+        # Non-200 HTTP response
+        if res.status != 200
+          err = ErrorParser.parse_connect_error(res.body, res.status, binding.rpc)
+
+          if can_refresh_and_retry?(err, refreshed_token)
+            refreshed_token = true
+            next
+          end
+
+          if can_retry?(err, binding, request, options, attempt)
+            sleep(@retry_policy.backoff_duration(attempt))
+            next
+          end
+
+          raise err
+        end
+
+        # Check gRPC-Web response payload
+        res_body = res.body
+        if @protocol == :grpc_web
+          frames = Envelopes.split(res_body)
+          trailer_frame = frames.find(&:trailers?)
+          if trailer_frame
+            headers = {}
+            trailer_frame.payload.each_line do |line|
+              if line.include?(":")
+                k, v = line.split(":", 2)
+                headers[k.strip.downcase] = v.strip
+              end
+            end
+            status_num = (headers["grpc-status"] || headers["Grpc-Status"]).to_i
+            if status_num != 0
+              err = ErrorParser.parse_grpc_trailers(headers, res.status, binding.rpc)
+              if can_refresh_and_retry?(err, refreshed_token)
+                refreshed_token = true
+                next
+              end
+              if can_retry?(err, binding, request, options, attempt)
+                sleep(@retry_policy.backoff_duration(attempt))
+                next
+              end
+              raise err
+            end
+          end
+
+          data_frame = frames.find(&:data?)
+          res_body = data_frame ? data_frame.payload : ""
+        end
+
+        return MessageCodec.deserialize(binding.response, res_body, @codec)
+      end
+    end
+
+    def server_stream(binding, request, options = {}, resume_policy = nil)
+      ct = ContentTypes.content_type_for(@protocol, @codec, streaming: true)
+      
+      stream_proc = proc do |cursor|
+        if cursor && !cursor.empty?
+          if request.respond_to?(:resume_cursor=)
+            request.resume_cursor = cursor
+          elsif request.respond_to?(:cursor=)
+            request.cursor = cursor
+          elsif request.is_a?(Hash)
+            request[:resume_cursor] = cursor
+          end
+        end
+
+        headers = { "content-type" => ct }
+        if (tok = current_token)
+          headers["authorization"] = "Bearer #{tok}"
+        end
+        if options[:headers]
+          options[:headers].each { |k, v| headers[k.to_s.downcase] = v.to_s }
+        end
+
+        body = MessageCodec.serialize(binding.request, request, @codec)
+        body = Envelopes.wrap(body, Envelopes::FLAG_DATA)
+
+        req = Request.new(
+          method: "POST",
+          path: "/#{binding.rpc}",
+          headers: headers,
+          body: body,
+          rpc: binding.rpc,
+        )
+
+        res = @transport.call_unary(req)
+        if res.status != 200
+          raise ErrorParser.parse_connect_error(res.body, res.status, binding.rpc)
+        end
+
+        Envelopes.split(res.body)
+      end
+
+      initial_frames = stream_proc.call(nil)
+      handle = ServerStreamHandle.new(initial_frames, &stream_proc)
+      handle.instance_variable_set(:@decode_proc, proc { |payload| MessageCodec.deserialize(binding.response, payload, @codec) })
+      handle
+    end
+
+    private
+
+    def current_token(force_refresh: false)
+      @token_source&.token(force_refresh: force_refresh)
+    end
+
+    def can_refresh_and_retry?(err, already_refreshed)
+      return false if already_refreshed || @token_source.nil?
+
+      if err.code == :unauthenticated || err.reason == :token_expired || err.http_status == 401
+        @token_source.refresh!
+        true
+      else
+        false
+      end
+    end
+
+    def can_retry?(err, binding, request, options, attempt)
+      max = options[:max_retries] || @retry_policy.max_retries
+      return false if attempt >= max + 1
+      return false unless err.retryable?
+
+      @retry_policy.retryable_call?(binding, request, options)
+    end
+  end
+end

data/lib/loams/pagination.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Loams
+  class PageIterator
+    include Enumerable
+
+    attr_reader :items_field, :next_page_token_field, :requested_tokens, :offered_tokens, :stopped_on_repeated_token
+
+    def initialize(fetch:, items_field:, next_page_token_field:)
+      @fetch = fetch
+      @items_field = items_field
+      @next_page_token_field = next_page_token_field
+      @requested_tokens = []
+      @offered_tokens = []
+      @stopped_on_repeated_token = false
+
+      validate_fields!
+    end
+
+    def each
+      return to_enum(:each) unless block_given?
+
+      started = false
+      next_token = ""
+      seen_tokens = []
+
+      loop do
+        token = started ? next_token : ""
+        if started && (token.nil? || token.empty?)
+          break
+        end
+
+        if started && seen_tokens.include?(token)
+          @stopped_on_repeated_token = true
+          break
+        end
+
+        @requested_tokens << token
+        seen_tokens << token if started
+        @offered_tokens << token if started
+
+        page = @fetch.call(token)
+        started = true
+
+        # Extract items
+        items = extract_field(page, @items_field) || []
+        items.each { |item| yield item }
+
+        # Extract next_page_token
+        next_token = extract_field(page, @next_page_token_field).to_s
+      end
+    end
+
+    private
+
+    def validate_fields!
+      # Validate items_field and next_page_token_field
+      if @items_field.respond_to?(:label) && @items_field.label != :repeated
+        raise ArgumentError, "#{@items_field.name} is not a repeated field"
+      end
+
+      if @items_field.respond_to?(:containing_type) && @next_page_token_field.respond_to?(:containing_type)
+        if @items_field.containing_type != @next_page_token_field.containing_type
+          raise ArgumentError, "fields declared on different messages"
+        end
+      end
+    end
+
+    def extract_field(message, field)
+      return nil if message.nil?
+
+      name = field.respond_to?(:name) ? field.name : field.to_s
+      json_name = field.respond_to?(:json_name) ? field.json_name : nil
+
+      if message.respond_to?(name)
+        message.public_send(name)
+      elsif json_name && message.respond_to?(json_name)
+        message.public_send(json_name)
+      elsif message.is_a?(Hash)
+        message[name] || (json_name ? message[json_name] : nil) || message[name.to_sym]
+      end
+    end
+  end
+end

data/lib/loams/retry.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Loams
+  class RetryPolicy
+    attr_reader :max_retries, :initial_backoff_ms, :max_backoff_ms
+
+    def initialize(max_retries: 3, initial_backoff_ms: 10, max_backoff_ms: 500)
+      @max_retries = max_retries
+      @initial_backoff_ms = initial_backoff_ms
+      @max_backoff_ms = max_backoff_ms
+    end
+
+    def retryable_call?(binding, request, options)
+      # If caller explicitly set retry policy
+      return false if options[:max_retries] == 0 || @max_retries == 0
+
+      # Check if this call is safe to retry
+      if binding.retry == :safe || binding.idempotency != :none
+        true
+      elsif binding.takes_idempotency_key?
+        # A mutation is ONLY retried if it carries an idempotency key!
+        key = if request.respond_to?(:idempotency_key)
+          request.idempotency_key
+        elsif request.is_a?(Hash)
+          request[:idempotency_key] || request["idempotency_key"]
+        end
+        !key.nil? && !key.empty?
+      else
+        false
+      end
+    end
+
+    def backoff_duration(attempt)
+      delay_ms = [@initial_backoff_ms * (2**(attempt - 1)), @max_backoff_ms].min
+      delay_ms / 1000.0
+    end
+  end
+end

data/lib/loams/streams.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require "loams/envelopes"
+require "loams/codec"
+
+module Loams
+  class ServerStreamHandle
+    attr_reader :frame_kinds, :heartbeats, :reconnects
+
+    def initialize(initial_iterator, &reconnect_proc)
+      @initial_iterator = initial_iterator
+      @reconnect_proc = reconnect_proc
+      @frame_kinds = []
+      @heartbeats = 0
+      @reconnects = 0
+    end
+
+    def messages
+      return to_enum(:messages) unless block_given?
+
+      last_cursor = nil
+      current_enum = @initial_iterator
+
+      loop do
+        begin
+          current_enum.each do |frame|
+            if frame.end_stream?
+              # Connect end frame: contains error if status != 0
+              handle_end_frame(frame.payload)
+              return
+            elsif frame.trailers?
+              # gRPC-Web trailers
+              handle_trailers_frame(frame.payload)
+              return
+            end
+
+            # Data frame
+            msg = decode_message(frame.payload)
+            kind = detect_kind(msg)
+            @frame_kinds << kind
+
+            cursor = detect_cursor(msg)
+            last_cursor = cursor if cursor && !cursor.empty?
+
+            if kind == "heartbeat"
+              @heartbeats += 1
+              next
+            end
+
+            yield msg
+          end
+          return
+        rescue LoamsError => e
+          if @reconnect_proc && e.retryable?
+            @reconnects += 1
+            current_enum = @reconnect_proc.call(last_cursor)
+          else
+            raise e
+          end
+        rescue StandardError => e
+          if @reconnect_proc
+            @reconnects += 1
+            current_enum = @reconnect_proc.call(last_cursor)
+          else
+            raise e
+          end
+        end
+      end
+    end
+
+    def count_frame(kind)
+      @frame_kinds << kind
+    end
+
+    def count_heartbeat
+      @heartbeats += 1
+    end
+
+    def record_reconnect
+      @reconnects += 1
+    end
+
+    private
+
+    def decode_message(payload)
+      @decode_proc ? @decode_proc.call(payload) : payload
+    end
+
+    def detect_kind(msg)
+      return "data" unless msg
+
+      h = msg.respond_to?(:to_h) ? msg.to_h : (msg.is_a?(Hash) ? msg : {})
+      [:snapshot, :heartbeat, :upsert, :remove, :update, :delete].each do |k|
+        val = h[k] || h[k.to_s]
+        return k.to_s if !val.nil?
+      end
+      "data"
+    end
+
+    def detect_cursor(msg)
+      return nil unless msg
+
+      h = msg.respond_to?(:to_h) ? msg.to_h : (msg.is_a?(Hash) ? msg : {})
+      c = h[:cursor] || h["cursor"] || h[:resume_token] || h["resume_token"]
+      c.to_s.empty? ? nil : c.to_s
+    end
+
+    def handle_end_frame(payload)
+      return if payload.nil? || payload.empty?
+
+      parsed = JSON.parse(payload) rescue nil
+      if parsed && parsed["error"]
+        raise ErrorParser.parse_connect_error(payload, 200)
+      end
+    end
+
+    def handle_trailers_frame(payload)
+      return if payload.nil? || payload.empty?
+
+      headers = {}
+      payload.each_line do |line|
+        if line.include?(":")
+          k, v = line.split(":", 2)
+          headers[k.strip.downcase] = v.strip
+        end
+      end
+
+      status = headers["grpc-status"]
+      if status && status.to_i != 0
+        raise ErrorParser.parse_grpc_trailers(headers)
+      end
+    end
+  end
+end

data/lib/loams/token_source.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Loams
+  class TokenSource
+    def initialize(token_or_callable = nil, &block)
+      @callable = block || (token_or_callable.respond_to?(:call) ? token_or_callable : nil)
+      @static_token = @callable ? nil : token_or_callable.to_s
+      @cached_token = nil
+      @mutex = Mutex.new
+    end
+
+    def token(force_refresh: false)
+      return @static_token if @static_token
+
+      @mutex.synchronize do
+        if @cached_token.nil? || force_refresh
+          @cached_token = @callable.call.to_s
+        end
+        @cached_token
+      end
+    end
+
+    def refresh!
+      token(force_refresh: true)
+    end
+  end
+end

data/lib/loams/transport.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+
+module Loams
+  Request = Struct.new(:method, :path, :headers, :body, :rpc, keyword_init: true)
+  Response = Struct.new(:status, :headers, :body, :stream, keyword_init: true)
+
+  class HttpTransport
+    def initialize(endpoint)
+      @endpoint = endpoint.to_s
+      # Trailing slashes trimmed by index: `sub(%r{/+$}, "")` backtracks
+      # quadratically on a long run of slashes (CodeQL rb/polynomial-redos).
+      cut = @endpoint.length
+      cut -= 1 while cut.positive? && @endpoint[cut - 1] == "/"
+      @endpoint = @endpoint[0, cut]
+      @uri = URI.parse(@endpoint)
+    end
+
+    def call_unary(request)
+      url = URI.join(@endpoint + "/", request.path.sub(%r{^/+}, ""))
+      http = Net::HTTP.new(url.host, url.port)
+      http.use_ssl = (url.scheme == "https")
+      http.read_timeout = 30
+      http.open_timeout = 5
+
+      req = Net::HTTPGenericRequest.new(
+        request.method || "POST",
+        true,
+        true,
+        url.request_uri,
+        request.headers || {}
+      )
+      req.body = request.body if request.body
+
+      res = http.request(req)
+
+      headers = {}
+      res.each_header { |k, v| headers[k.downcase] = v }
+
+      Response.new(
+        status: res.code.to_i,
+        headers: headers,
+        body: res.body,
+      )
+    end
+
+    def call_stream(request, &block)
+      url = URI.join(@endpoint + "/", request.path.sub(%r{^/+}, ""))
+      http = Net::HTTP.new(url.host, url.port)
+      http.use_ssl = (url.scheme == "https")
+      http.read_timeout = 60
+      http.open_timeout = 5
+
+      req = Net::HTTPGenericRequest.new(
+        request.method || "POST",
+        true,
+        true,
+        url.request_uri,
+        request.headers || {}
+      )
+      req.body = request.body if request.body
+
+      if block_given?
+        http.request(req) do |res|
+          headers = {}
+          res.each_header { |k, v| headers[k.downcase] = v }
+          yield res.code.to_i, headers, res
+        end
+      else
+        res = http.request(req)
+        headers = {}
+        res.each_header { |k, v| headers[k.downcase] = v }
+        Response.new(
+          status: res.code.to_i,
+          headers: headers,
+          body: res.body,
+        )
+      end
+    end
+  end
+end

data/lib/loams/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Loams
+  VERSION = "0.1.0"
+end

data/lib/loams/wire.rb

@@ -0,0 +1,110 @@
+# The protobuf wire format, read and written by hand.
+#
+# Two things need it and neither is the message codec: the **custom
+# options** the protos carry (the `loams.options.v1.module` and
+# `loams.options.v1.facade` annotations, which `buf build` writes as
+# extension fields on `ServiceOptions` and `MethodOptions` and which the
+# descriptor set therefore carries as unknown fields), and the **Connect
+# and gRPC-Web frames** a stream is made of. Both are length-delimited
+# fields with a varint tag, so one reader serves them.
+#
+# The reader is deliberately strict: a tag that names a wire type it
+# cannot skip is a refusal, not a guess, because a mis-read byte here is
+# a module name invented out of nothing.
+module Loams
+  module Wire
+    # A varint read from +bytes+ at +pos+. Returns [value, next_pos].
+    def self.read_varint(bytes, pos)
+      value = 0
+      shift = 0
+      loop do
+        b = bytes.getbyte(pos)
+        raise ArgumentError, "truncated varint at #{pos}" if b.nil?
+        pos += 1
+        value |= (b & 0x7f) << shift
+        break if (b & 0x80).zero?
+        shift += 7
+        raise ArgumentError, "varint too long at #{pos}" if shift > 63
+      end
+      [value, pos]
+    end
+
+    # A varint written to +io+.
+    def self.write_varint(io, value)
+      until value < 0x80
+        io.write(((value & 0x7f) | 0x80).chr)
+        value >>= 7
+      end
+      io.write(value.chr)
+    end
+
+    # The tag for a field: the field number in the high bits, the wire
+    # type in the low three.
+    def self.tag(field_number, wire_type)
+      (field_number << 3) | wire_type
+    end
+
+    # Every field in +bytes+, as [field_number, wire_type, value] where
+    # value is an Integer for a varint and a String for a length-delimited
+    # field. Unknown wire types (1, 5 — fixed64, fixed32) are skipped by
+    # length only when the caller can say how long they are, which it
+    # cannot from a tag alone, so they are refused.
+    def self.each_field(bytes)
+      return to_enum(:each_field, bytes) unless block_given?
+      pos = 0
+      while pos < bytes.bytesize
+        tag, pos = read_varint(bytes, pos)
+        field = tag >> 3
+        wire_type = tag & 7
+        case wire_type
+        when 0
+          value, pos = read_varint(bytes, pos)
+          yield field, wire_type, value
+        when 2
+          len, pos = read_varint(bytes, pos)
+          raise ArgumentError, "length #{len} overruns #{bytes.bytesize} at #{pos}" if pos + len > bytes.bytesize
+          value = bytes.byteslice(pos, len)
+          pos += len
+          yield field, wire_type, value
+        when 1
+          raise ArgumentError, "fixed64 field #{field} cannot be skipped" if pos + 8 > bytes.bytesize
+          pos += 8
+        when 5
+          raise ArgumentError, "fixed32 field #{field} cannot be skipped" if pos + 4 > bytes.bytesize
+          pos += 4
+        else
+          raise ArgumentError, "wire type #{wire_type} on field #{field} is a group, which this corpus does not carry"
+        end
+      end
+    end
+
+    # The first length-delimited field +field_number+ in +bytes+, or nil.
+    def self.find_bytes(bytes, field_number)
+      each_field(bytes) do |field, wire_type, value|
+        return value if field == field_number && wire_type == 2
+      end
+      nil
+    end
+
+    # The first varint field +field_number+ in +bytes+, or nil.
+    def self.find_varint(bytes, field_number)
+      each_field(bytes) do |field, wire_type, value|
+        return value if field == field_number && wire_type == 0
+      end
+      nil
+    end
+
+    # A length-delimited field written to +io+.
+    def self.write_bytes(io, field_number, payload)
+      write_varint(io, tag(field_number, 2))
+      write_varint(io, payload.bytesize)
+      io.write(payload)
+    end
+
+    # A varint field written to +io+.
+    def self.write_varint_field(io, field_number, value)
+      write_varint(io, tag(field_number, 0))
+      write_varint(io, value)
+    end
+  end
+end