kafka_syrup 0.9.0

data/lib/kafka_syrup/protocol.rb

@@ -0,0 +1,22 @@
+%w[
+  errors
+  base
+  request
+  response
+  message
+  message_set
+  metadata_request
+  metadata_response
+  produce_request
+  produce_response
+  fetch_request
+  fetch_response
+  offset_request
+  offset_response
+].each{ |file| require "kafka_syrup/protocol/#{file}" }
+
+module KafkaSyrup
+  module Protocol
+    REPLICA_ID = -1
+  end
+end

data/lib/kafka_syrup/protocol/base.rb

@@ -0,0 +1,41 @@
+module KafkaSyrup
+  module Protocol
+    class Base
+      include Utils
+
+      def initialize(*args, &block)
+        if(io = args.first).respond_to?(:read)
+          decode(io, &block)
+        else
+          load_args(defaults)
+          load_args(*args)
+        end
+      end
+
+      def defaults
+        {}
+      end
+
+      def config
+        @config ||= ::KafkaSyrup.config
+      end
+
+      def encode(&block)
+        encoded = block_given? ? yield : ""
+
+        [
+          E.write_int32(encoded.length),
+          encoded
+        ].join
+      end
+
+      def decode(io)
+        E.read_int32(io) # Total length (ignored)
+      end
+
+      def ==(obj)
+        obj.encode.bytes == encode.bytes
+      end
+    end
+  end
+end

data/lib/kafka_syrup/protocol/errors.rb

@@ -0,0 +1,32 @@
+module KafkaSyrup
+  class KafkaResponseError < Error; end;
+
+  module KafkaResponseErrors
+    ERRORS = {
+      -1 => 'Unknown',
+      1 => 'OffsetOutOfRange',
+      2 => 'InvalidMessage',
+      3 => 'UnknownTopicOrPartition',
+      4 => 'InvalidMessageSize',
+      5 => 'LeaderNotAvailable',
+      6 => 'NotLeaderForPartition',
+      7 => 'RequestTimedOut',
+      8 => 'BrokerNotAvailable',
+      9 => 'ReplicaNotAvailable',
+      10 => 'MessageSizeTooLarge',
+      11 => 'StaleControllerEpochCode',
+      12 => 'OffsetMetadataTooLargeCode',
+      14 => 'OffsetsLoadInProgressCode',
+      15 => 'ConsumerCoordinatorNotAvailableCode',
+      16 => 'NotCoordinatorForConsumerCode'
+    }
+
+    ERRORS.values.each{ |val| const_set val, Class.new(KafkaResponseError) }
+
+    class << self
+      def raise_from_code(code, msg = nil)
+        raise const_get(ERRORS[code]).new(msg) if ERRORS.keys.include?(code)
+      end
+    end
+  end
+end

data/lib/kafka_syrup/protocol/fetch_request.rb

@@ -0,0 +1,60 @@
+module KafkaSyrup
+  module Protocol
+    class FetchRequest < Request
+      self.api_key = 1
+
+      attr_accessor :max_wait_time, :min_bytes, :max_bytes, :topics
+
+      def defaults
+        {
+          max_wait_time: config.consume_max_wait_time,
+          min_bytes: config.consume_min_bytes,
+          max_bytes: config.consume_max_bytes,
+          topics: []
+        }
+      end
+
+      def encode
+        super do
+          [
+            E.write_int32(REPLICA_ID),
+            E.write_int32(max_wait_time),
+            E.write_int32(min_bytes),
+            E.write_array(topics)
+          ].join
+        end
+      end
+
+      def add_topic(name)
+        topic = Topic.new(name, [], max_bytes)
+        topics << topic
+        topic
+      end
+
+      Topic = Struct.new(:name, :partitions, :max_bytes) do
+        def add_partition(id, offset)
+          partition = Partition.new(id, offset, max_bytes)
+          partitions << partition
+          partition
+        end
+
+        def encode
+          [
+            E.write_string(name),
+            E.write_array(partitions)
+          ].join
+        end
+      end
+
+      Partition = Struct.new(:id, :offset, :max_bytes) do
+        def encode
+          [
+            E.write_int32(id),
+            E.write_int64(offset),
+            E.write_int32(max_bytes)
+          ].join
+        end
+      end
+    end
+  end
+end

data/lib/kafka_syrup/protocol/fetch_response.rb

@@ -0,0 +1,84 @@
+module KafkaSyrup
+  module Protocol
+    class FetchResponse < Response
+      attr_accessor :topics
+
+      def defaults
+        { topics: [] }
+      end
+
+      def add_topic(name)
+        topic = Topic.new(name, [])
+        topics << topic
+        topic
+      end
+
+      def encode
+        super do
+          E.write_array(topics)
+        end
+      end
+
+      def decode(io, &block)
+        super
+        self.topics = E.read_array(io){ |input| Topic.decode(input, &block) }
+        topics.flat_map(&:partitions).map(&:code).each(&KafkaResponseErrors.method(:raise_from_code))
+      end
+
+      Topic = Struct.new(:name, :partitions) do
+        def add_partition(id, code, highwater_offset)
+          partition = Partition.new(id, code, highwater_offset, MessageSet.new)
+          partitions << partition
+          partition
+        end
+
+        def encode
+          [
+            E.write_string(name),
+            E.write_array(partitions)
+          ].join
+        end
+
+        def self.decode(io, &block)
+          new(
+            E.read_string(io),                                          # Name
+            E.read_array(io){ |input| Partition.decode(input, &block) } # Partitions
+          )
+        end
+      end
+
+      Partition = Struct.new(:id, :code, :highwater_offset, :message_set) do
+        def messages
+          message_set.messages
+        end
+
+        def add_message(value = nil, opts = {})
+          m = Message.new(opts.merge(value: value))
+          message_set.messages << m
+          m
+        end
+
+        def encode
+          encoded = message_set.encode
+          [
+            E.write_int32(id),
+            E.write_int16(code),
+            E.write_int64(highwater_offset),
+            E.write_int32(encoded.length),
+            encoded
+          ].join
+        end
+
+        def self.decode(io, &block)
+          partition = new
+          partition.id = E.read_int32(io)
+          partition.code = E.read_int16(io)
+          partition.highwater_offset = E.read_int64(io)
+          length = E.read_int32(io)
+          partition.message_set = MessageSet.new(io, length, &block)
+          partition
+        end
+      end
+    end
+  end
+end

data/lib/kafka_syrup/protocol/message.rb

@@ -0,0 +1,58 @@
+require 'zlib'
+
+module KafkaSyrup
+  module Protocol
+    class Message
+      include Utils
+
+      MAGIC_BYTE = 0
+
+      attr_accessor :key, :value, :offset
+
+      def initialize(*args)
+        if (io = args.first).respond_to?(:read)
+          opts = { check_crc: false }
+          opts.merge(args.last) if args.last.is_a?(Hash)
+          load_args(opts)
+
+          crc = E.read_int32(io)
+          E.read_int8(io)     # Magic Byte (ignored)
+          E.read_int8(io)     # Compression (ignored)
+          self.key = E.read_bytes(io)
+          self.value = E.read_bytes(io)
+          # TODO Verify CRC
+        end
+
+        load_args(*args)
+      end
+
+      def encode
+        [
+          E.write_int32(crc),
+          encoded
+        ].join
+      end
+
+      def encoded
+        @encoded ||= [
+          E.write_int8(MAGIC_BYTE),
+          E.write_int8(0), # Currently don't support compression
+          E.write_bytes(key),
+          E.write_bytes(value)
+        ].join
+      end
+
+      def crc
+        @crc ||= Zlib.crc32(encoded)
+      end
+
+      def key
+        @key.to_s.empty? ? nil : @key
+      end
+
+      def ==(obj)
+        obj.encode == encode
+      end
+    end
+  end
+end

data/lib/kafka_syrup/protocol/message_set.rb

@@ -0,0 +1,53 @@
+module KafkaSyrup
+  module Protocol
+    class MessageSet
+      include Utils
+
+      attr_accessor :messages
+
+      def initialize(*args)
+        io, total_length, *_ = args
+        if io.respond_to?(:read)
+          total_length = 0 unless total_length.is_a?(Fixnum)
+
+          read_length = 0
+
+          self.messages = []
+
+          while read_length < total_length && !io.eof?
+            offset = E.read_int64(io)
+            read_length += 8
+            msg_length = E.read_int32(io)
+            read_length += 4
+            msg = Message.new(io, offset: offset, length: msg_length) rescue nil
+            messages << msg
+            read_length += msg_length
+            yield(msg) if block_given? && msg
+          end
+        else
+          load_args(defaults)
+          load_args(*args)
+        end
+      end
+
+      def defaults
+        { messages: [] }
+      end
+
+      def encode
+        messages.map{ |msg|
+          encoded = msg.encode
+          [
+            E.write_int64(msg.offset.to_i),
+            E.write_int32(encoded.length),
+            encoded
+          ].join
+        }.join
+      end
+
+      def ==(obj)
+        obj.encode == encode
+      end
+    end
+  end
+end

data/lib/kafka_syrup/protocol/metadata_request.rb

@@ -0,0 +1,23 @@
+module KafkaSyrup
+  module Protocol
+    class MetadataRequest < Request
+      self.api_key = 3
+
+      attr_accessor :topics
+
+      def initialize(*args)
+        opts = args.last.is_a?(Hash) ? args.pop : {}
+
+        load_args(opts)
+
+        self.topics = args
+      end
+
+      def encode
+        super do
+          E.write_array(topics, &E.method(:write_string))
+        end
+      end
+    end
+  end
+end

data/lib/kafka_syrup/protocol/metadata_response.rb

@@ -0,0 +1,105 @@
+module KafkaSyrup
+  module Protocol
+    class MetadataResponse < Response
+      attr_accessor :brokers, :topics
+
+      def defaults
+        { brokers: [], topics: [] }
+      end
+
+      def add_broker(node, host, port)
+        broker = Broker.new(node, host, port)
+        brokers << broker
+        broker
+      end
+
+      def add_topic(code, name)
+        topic = Topic.new(code, name, [])
+        topics << topic
+        topic
+      end
+
+      def encode
+        super do
+          [
+            E.write_array(brokers),
+            E.write_array(topics)
+          ].join
+        end
+      end
+
+      def decode(io)
+        super
+        self.brokers = E.read_array(io, &Broker.method(:decode))
+        self.topics = E.read_array(io, &Topic.method(:decode))
+
+        topics.map(&:code).each(&KafkaResponseErrors.method(:raise_from_code))
+        topics.flat_map(&:partitions).map(&:code).each(&KafkaResponseErrors.method(:raise_from_code))
+      end
+
+      Broker = Struct.new(:node, :host, :port) do
+        def encode
+          [
+            E.write_int32(node),
+            E.write_string(host),
+            E.write_int32(port)
+          ].join
+        end
+
+        def self.decode(io)
+          new(
+            E.read_int32(io),   # Node
+            E.read_string(io),  # Host
+            E.read_int32(io)    # Port
+          )
+        end
+      end
+
+      Topic = Struct.new(:code, :name, :partitions) do
+        def add_partition(p_code, id, leader, replicas, isr)
+          partition = Partition.new(p_code, id, leader, replicas, isr)
+          partitions << partition
+          partition
+        end
+
+        def encode
+          [
+            E.write_int16(code),
+            E.write_string(name),
+            E.write_array(partitions)
+          ].join
+        end
+
+        def self.decode(io)
+          new(
+            E.read_int16(io),                            # Error Code
+            E.read_string(io),                           # Name
+            E.read_array(io, &Partition.method(:decode)) # Partitions
+          )
+        end
+      end
+
+      Partition = Struct.new(:code, :id, :leader, :replicas, :isr) do
+        def encode
+          [
+            E.write_int16(code),
+            E.write_int32(id),
+            E.write_int32(leader),
+            E.write_array(replicas, &E.method(:write_int32)),
+            E.write_array(isr, &E.method(:write_int32))
+          ].join
+        end
+
+        def self.decode(io)
+          new(
+            E.read_int16(io),                         # Error Code
+            E.read_int32(io),                         # ID
+            E.read_int32(io),                         # Leader
+            E.read_array(io, &E.method(:read_int32)), # Replicas
+            E.read_array(io, &E.method(:read_int32))  # ISR
+          )
+        end
+      end
+    end
+  end
+end