rom-kafka 0.0.1

data/lib/rom/kafka/connection/producer.rb

@@ -0,0 +1,114 @@
+# encoding: utf-8
+
+module ROM::Kafka
+
+  class Connection
+
+    # The producer-specific connection to Kafka cluster
+    #
+    # It is wrapped around `Poseidon::Producer` driver, and responsible for
+    # adopting poseidon API to ROM::Gateway via [#initializer] and [#publish]
+    # methods.
+    #
+    # ROM::Kafka producer deals with tuples, hiding poseidon-specific
+    # implementation of messages from the rest of the gem.
+    #
+    # @api private
+    #
+    class Producer < Connection
+
+      # The 'poseidon' class describing a producer
+      #
+      # @return [Class]
+      #
+      DRIVER = Poseidon::Producer
+
+      # The 'poseidon' class describing a message acceptable by producer
+      #
+      # @return [Class]
+      #
+      MESSAGE = Poseidon::MessageToSend
+
+      # Attributes, acceptable by the `Poseidon::Producer` driver
+      attribute :partitioner
+      attribute :type, default: :sync
+      attribute :compression_codec
+      attribute :metadata_refresh_interval_ms
+      attribute :max_send_retries
+      attribute :retry_backoff_ms
+      attribute :required_acks
+      attribute :ack_timeout_ms
+      attribute :socket_timeout_ms
+
+      # @!attribute [r] connection
+      #
+      # @return [ROM::Kafka::Connections::Producer::DRIVER] driver to Kafka
+      #
+      attr_reader :connection
+
+      # Initializes a producer connection
+      #
+      # The initializer is attributes-agnostic. This means it doesn't validate
+      # attributes, but skips unused.
+      #
+      # @option options [#to_s] :client_id
+      #   A required unique id used to indentify the Kafka client.
+      # @option options [Array<String>] :brokers
+      #   A list of seed brokers to find a lead broker to fetch messages from.
+      # @option options [Proc, nil] :partitioner
+      #   A proc used to provide partition from given key.
+      # @option options [:gzip, :snappy, nil] :compression_codec (nil)
+      #   Type of compression to be used.
+      # @option options [Integer] :metadata_refresh_interval_ms (600_000)
+      #   How frequently the topic metadata should be updated (in milliseconds).
+      # @option options [Integer] :max_send_retries (3)
+      #   Number of times to retry sending of messages to a leader.
+      # @option options [Integer] :retry_backoff_ms (100)
+      #   An amount of time (in milliseconds) to wait before refreshing
+      #   the metadata after we are unable to send messages.
+      # @option options [Integer] :required_acks (0)
+      #   A number of acks required per request.
+      # @option options [Integer] :ack_timeout_ms (1_500)
+      #   How long the producer waits for acks.
+      # @option options [Integer] :socket_timeout_ms (10_000)
+      #   How long the producer/consumer socket waits for any reply from server.
+      #
+      def initialize(options)
+        super # takes declared attributes only, skipping brokers and client_id
+        brokers    = options.fetch(:brokers)
+        client     = options.fetch(:client_id)
+        @connection = DRIVER.new(brokers, client, attributes)
+      end
+
+      # Sends tuples to the underlying connection
+      #
+      # Stringifies non-empty hash values to conform to 'poseidon' API.
+      #
+      # @param [Array<Hash>] data
+      #
+      # @return [Array<Hash{Symbol => String, nil}>]
+      #   The list of published tuples
+      #
+      def publish(*data)
+        tuples = data.flatten.map(&method(:stringify_keys))
+        @connection.send_messages tuples.map(&method(:message))
+
+        tuples
+      end
+
+      private
+
+      def stringify_keys(tuple)
+        keys = [:value, :topic, :key]
+        Hash[keys.zip(tuple.values_at(*keys).map { |v| v.to_s if v })]
+      end
+
+      def message(tuple)
+        MESSAGE.new(*tuple.values_at(:topic, :value, :key))
+      end
+
+    end # class Producer
+
+  end # class Connection
+
+end # module ROM::Kafka

data/lib/rom/kafka/create.rb

@@ -0,0 +1,75 @@
+# encoding: utf-8
+
+module ROM::Kafka
+
+  # The namespace for Kafka-specific ROM commands
+  #
+  module Commands
+
+    # The Kafka-specific implementation of ROM::Commands::Create
+    #
+    # @example
+    #   ROM.use(:auto_registration)
+    #   ROM.setup(:kafka, "localhost:9092")
+    #
+    #   class Users < ROM::Relation[:kafka]
+    #     dataset :users
+    #   end
+    #
+    #   class GreetUsers < ROM::Commands::Create[:kafka]
+    #     relation :users
+    #     register_as :greet
+    #   end
+    #
+    #   rom = ROM.finalize.env
+    #   greet = rom.commands(:users).greet
+    #   greet.with(key: "greetings").call "Hi!"
+    #   # => [{ value: "Hi!", topic: "users", key: "greetings" }]
+    #
+    class Create < ROM::Commands::Create
+
+      adapter :kafka
+      option  :key, reader: true
+
+      # Sends messages to the current topic/partition of Kafka
+      #
+      # @param [#to_s, Array<#to_s>] messages
+      #
+      # @return [Array<Hash>]
+      #
+      def execute(*messages)
+        tuples = messages.flatten.map(&method(:tuple))
+        producer.publish(*tuples)
+      end
+
+      # Returns a new command where `:key` option is updated
+      #
+      # @param [Hash] options
+      # @options options [Object] :key
+      #   The key to be used by Kafka to define a partition
+      #
+      # @return [ROM::Kafka::Commands::Create]
+      #
+      def with(options)
+        self.class.new relation, key: options.fetch(:key)
+      end
+
+      private
+
+      def producer
+        dataset.producer
+      end
+
+      def dataset
+        relation.dataset
+      end
+
+      def tuple(text)
+        { value: text.to_s, topic: dataset.topic, key: key }
+      end
+
+    end # class Create
+
+  end # module Commands
+
+end # module ROM::Kafka

data/lib/rom/kafka/dataset.rb

@@ -0,0 +1,132 @@
+# encoding: utf-8
+
+module ROM::Kafka
+
+  # The dataset describes Kafka topic
+  #
+  # @api private
+  #
+  class Dataset
+
+    extend AttributesDSL
+    include Enumerable
+
+    # Customizable attributes for a consumer connection
+    attribute :partition, default: 0
+    attribute :offset,    default: 0
+    attribute :limit,     default: 0
+    attribute :min_bytes
+    attribute :max_bytes
+    attribute :max_wait_ms
+
+    # @!attribute [r] gateway
+    #
+    # @return [ROM::Kafka::Gateway]
+    #   The back reference to the gateway, that provided the dataset
+    #
+    attr_reader :gateway
+
+    # @!attribute [r] topic
+    #
+    # @return [String] The name of the topic, described by the dataset
+    #
+    attr_reader :topic
+
+    # @!attribute [r] producer
+    #
+    # @return [ROM::Kafka::Connection::Producer]
+    #   The producer connection to Kafka brokers, defined by a gateway.
+    #   It is stored for being used by a `Create` command.
+    #
+    attr_reader :producer
+
+    # @!attribute [r] consumer
+    #
+    # @return [ROM::Kafka::Connection::Consumer]
+    #   The consumer connection to Kafka brokers, used to fetch messages
+    #   via [#each] method call.
+    #
+    attr_reader :consumer
+
+    # Initializes the dataset with a gateway and topic name
+    #
+    # Attributes are set by default from a gateway. Later you can create
+    # new dataset for the same gateway and topic, but with attributes,
+    # updated via [#using] method.
+    #
+    # @param [ROM::Kafka::Gateway] gateway
+    # @param [String] topic
+    #
+    # @option options [Integer] :partition (0)
+    #   A partition number to fetch messages from
+    # @option options [Integer] :offset (0)
+    #   An initial offset to start fetching from.
+    # @option options [Integer] :min_bytes (1)
+    #   A smallest amount of data the server should send.
+    #   (By default send us data as soon as it is ready).
+    # @option options [Integer] :max_bytes (1_048_576)
+    #   A maximum number of bytes to fetch by consumer (1MB by default).
+    # @option options [Integer] :max_wait_ms (100)
+    #   How long to block until the server sends data.
+    #   NOTE: This is only enforced if min_bytes is > 0.
+    #
+    def initialize(gateway, topic, options = {})
+      super gateway.attributes.merge(options)
+      @topic    = topic.to_s
+      @gateway  = gateway
+      @producer = gateway.producer
+      @consumer = prepare_consumer
+    end
+
+    # Returns a new dataset with updated consumer attributes
+    #
+    # @param [Hash] options The part of attributes to be updated
+    #
+    # @return [ROM::Kafka::Dataset]
+    #
+    def using(options)
+      self.class.new(gateway, topic, attributes.merge(options))
+    end
+
+    # Returns the enumerator to iterate via tuples, fetched from a [#consumer].
+    #
+    # If a `limit` of messages is set, iterator stops after achieving it.
+    #
+    # @param [Proc] block
+    #
+    # @yieldparam [Hash] tuple
+    #
+    # @return [Enumerator<Hash{Symbol => String, Integer}>]
+    #
+    def each(&block)
+      return to_enum unless block_given?
+      limit.equal?(0) ? unlimited_each(&block) : limited_each(&block)
+    end
+
+    private
+
+    def prepare_consumer
+      Connection::Consumer.new consumer_options
+    end
+
+    def consumer_options
+      attributes.merge(
+        topic: topic,
+        client_id: gateway.client_id,
+        brokers: gateway.brokers
+      )
+    end
+
+    def unlimited_each
+      enum = consumer.each
+      loop { yield(enum.next) }
+    end
+
+    def limited_each
+      enum = consumer.each
+      limit.times { yield(enum.next) }
+    end
+
+  end # class Dataset
+
+end # module ROM::Kafka

data/lib/rom/kafka/gateway.rb

@@ -0,0 +1,165 @@
+# encoding: utf-8
+
+module ROM::Kafka
+
+  # Describes the gateway to Kafka
+  #
+  # The gateway has 3 responsibilities:
+  # - registers the datasets describing various topics and partitions
+  # - instantiates the producer connection to Kafka brokers that doesn't
+  #   depend on a specific topic/partition settings
+  # - stores settings for the consumer connections to Kafka, that
+  #   depends on a specific topic/partition/offset
+  #
+  # Every dataset uses the same producer connection (defined by gateway) and
+  # individual consumer's one. The consumer's connection is reloaded
+  # every time the topic, partition or current offset is changed by a relation.
+  #
+  class Gateway < ROM::Gateway
+
+    extend AttributesDSL
+
+    # Attributes used by both producer and consumer
+    attribute :client_id, required: true, &:to_s
+    attribute :brokers
+
+    # Producer-specific attributes
+    attribute :partitioner
+    attribute :compression_codec
+    attribute :metadata_refresh_interval_ms, default: 600_000
+    attribute :max_send_retries, default: 3
+    attribute :retry_backoff_ms, default: 100
+    attribute :required_acks, default: 0
+    attribute :ack_timeout_ms, default: 1_500
+    attribute :socket_timeout_ms, default: 10_000
+
+    # Consumer-specific attributes
+    attribute :min_bytes, default: 1
+    attribute :max_bytes, default: 1_048_576
+    attribute :max_wait_ms, default: 100
+
+    # Initializes the gateway to Kafka broker(s).
+    #
+    # The initializer is attributes-agnostic. This means it doesn't validate
+    # attributes, but skips unused.
+    #
+    # @example Initialize a producer's gateway to Kafka
+    #   gateway = Gateway.new(
+    #     hosts: ["127.0.0.1", "127.0.0.2:9093"],
+    #     port: 9092,
+    #     client_id: :my_user,
+    #     compression-codec: :gzip
+    #   )
+    #   gateway.brokers # => ["127.0.0.1:9092", "127.0.0.2:9093"]
+    #
+    # @example Alternative syntax
+    #   gateway = Gateway.new(
+    #     "127.0.0.1:9092",
+    #     "127.0.0.2:9093",
+    #     client_id: :my_user,
+    #     compression-codec: :gzip
+    #   )
+    #   gateway.brokers # => ["127.0.0.1:9092", "127.0.0.2:9093"]
+    #
+    # @example Mixed syntax
+    #   gateway = Gateway.new(
+    #     "127.0.0.1:9092",
+    #     hosts: ["127.0.0.2"]
+    #     port: 9093,
+    #     client_id: :my_user,
+    #     min_bytes: 1024 # wait until 1Kb of messages is prepared
+    #   )
+    #   gateway.brokers # => ["127.0.0.1:9092", "127.0.0.2:9093"]
+    #
+    # @param [nil, String, Array<String>] addresses
+    #   The address(es) of broker(s) to connect (optional).
+    #   Brokers can be alternatively set with `:hosts` and `:port` options.
+    #
+    # @option options [#to_s] :client_id
+    #   A required unique id used to indentify the Kafka client.
+    # @option options [String, Array<String>] :hosts
+    #   A host or list of hosts in the form "host1:port1" or "host1".
+    #   In case of a consumer, only the first host is actually used.
+    # @option options [Integer] :port
+    #   The port shared by all hosts.
+    #
+    # @option options [Proc, nil] :partitioner
+    #   A proc used to provide partition from given key.
+    # @option options [:gzip, :snappy, nil] :compression_codec (nil)
+    #   Type of compression to be used.
+    # @option options [Integer] :metadata_refresh_interval_ms (600_000)
+    #   How frequently the topic metadata should be updated (in milliseconds).
+    # @option options [Integer] :max_send_retries (3)
+    #   Number of times to retry sending of messages to a leader.
+    # @option options [Integer] :retry_backoff_ms (100)
+    #   An amount of time (in milliseconds) to wait before refreshing
+    #   the metadata after we are unable to send messages.
+    # @option options [Integer] :required_acks (0)
+    #   A number of acks required per request.
+    # @option options [Integer] :ack_timeout_ms (1_500)
+    #   How long the producer waits for acks.
+    # @option options [Integer] :socket_timeout_ms (10_000)
+    #   How long the producer/consumer socket waits for any reply from server.
+    #
+    # @option options [Integer] :offset
+    #   An initial offset to start fetching from.
+    # @option options [Integer] :min_bytes (1)
+    #   A smallest amount of data the server should send.
+    #   (By default send us data as soon as it is ready).
+    # @option options [Integer] :max_bytes (1_048_576)
+    #   A maximum number of bytes to fetch by consumer (1MB by default).
+    # @option options [Integer] :max_wait_ms (100)
+    #   How long to block until the server sends data.
+    #   NOTE: This is only enforced if min_bytes is > 0.
+    #
+    def initialize(*addresses)
+      options = Hash[addresses.pop]
+      brokers = Brokers.new(addresses, options).to_a
+      super options.merge(brokers: brokers) # prepares #attributes
+
+      @producer = Connection::Producer.new(attributes)
+      @datasets = {}
+    end
+
+    # @!attribute [r] producer
+    #
+    # @return [ROM::Kafka::Producer] the producer's connection to Kafka brockers
+    #
+    attr_reader :producer
+
+    # Returns the registered dataset by topic
+    #
+    # @param [#to_sym] topic
+    #
+    # @return [ROM::Kafka::Dataset]
+    #
+    def [](topic)
+      @datasets[topic.to_sym]
+    end
+
+    # Registers the dataset by topic
+    #
+    # By default the dataset is registered with 0 partition and 0 offset.
+    # That settings can be changed from either relation of a command.
+    #
+    # @param [#to_sym] topic
+    #
+    # @return [self] itself
+    #
+    def dataset(topic)
+      @datasets[topic.to_sym] ||= Dataset.new(self, topic)
+    end
+
+    # Checks whether a dataset is registered by topic
+    #
+    # @param [#to_sym] topic
+    #
+    # @return [Boolean]
+    #
+    def dataset?(topic)
+      self[topic] ? true : false
+    end
+
+  end # class Gateway
+
+end # module ROM::Kafka