rom-kafka 0.0.1

data/config/metrics/cane.yml

@@ -0,0 +1,5 @@
+---
+abc_max: "10"
+line_length: "80"
+no_doc: "y"
+no_readme: "y"

data/config/metrics/churn.yml

@@ -0,0 +1,6 @@
+---
+ignore_files:
+- spec
+- config
+minimum_churn_count: 0
+start_date: "1 year ago"

data/config/metrics/flay.yml

@@ -0,0 +1,2 @@
+---
+minimum_score: 9

data/config/metrics/metric_fu.yml

@@ -0,0 +1,14 @@
+---
+folders: # The list of folders to be used by any metric.
+- lib
+metrics: # The list of allowed metrics. The other metrics are disabled.
+- cane
+- churn
+- flay
+- flog
+- reek
+- roodi
+- saikuro
+format: html
+output: tmp/metric_fu
+verbose: false

data/config/metrics/reek.yml

@@ -0,0 +1 @@
+---

data/config/metrics/roodi.yml

@@ -0,0 +1,24 @@
+---
+AssignmentInConditionalCheck:
+CaseMissingElseCheck:
+ClassLineCountCheck:
+  line_count: 200
+ClassNameCheck:
+  pattern: !ruby/regexp /^[A-Z][a-zA-Z0-9]*$/
+ClassVariableCheck:
+CyclomaticComplexityBlockCheck:
+  complexity: 3
+CyclomaticComplexityMethodCheck:
+  complexity: 5
+EmptyRescueBodyCheck:
+ForLoopCheck:
+MethodLineCountCheck:
+  line_count: 6
+MethodNameCheck:
+  pattern: !ruby/regexp /^[\||\^|\&|\!]$|^[_a-z<>=\[|+-\/\*`]+[_a-z0-9_<>=~@\[\]]*[=!\?]?$/
+ModuleLineCountCheck:
+  line_count: 200
+ModuleNameCheck:
+  pattern: !ruby/regexp /^[A-Z][a-zA-Z0-9]*$/
+ParameterNumberCheck:
+  parameter_count: 5

data/config/metrics/rubocop.yml

@@ -0,0 +1,71 @@
+---
+AllCops:
+  Exclude:
+    - '**/db/schema.rb'
+
+Lint/HandleExceptions:
+  Exclude:
+    - '**/*_spec.rb'
+
+Lint/RescueException:
+  Exclude:
+    - '**/*_spec.rb'
+
+Style/AccessorMethodName:
+  Exclude:
+    - '**/*_spec.rb'
+
+Style/AsciiComments:
+  Enabled: false
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/EmptyLinesAroundBlockBody:
+  Enabled: false
+
+Style/EmptyLinesAroundClassBody:
+  Enabled: false
+
+Style/EmptyLinesAroundMethodBody:
+  Enabled: false
+
+Style/EmptyLinesAroundModuleBody:
+  Enabled: false
+
+Style/EmptyLineBetweenDefs:
+  Enabled: false
+
+Style/FileName:
+  Enabled: false
+
+Style/NumericLiterals:
+  Enabled: false
+
+Style/RaiseArgs:
+  EnforcedStyle: compact
+
+Style/SingleLineMethods:
+  Exclude:
+    - '**/*_spec.rb'
+
+Style/SingleSpaceBeforeFirstArg:
+  Enabled: false
+
+Style/SpecialGlobalVars:
+  Exclude:
+    - '**/Gemfile'
+    - '**/*.gemspec'
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: double_quotes
+
+Style/TrivialAccessors:
+  Exclude:
+    - '**/*_spec.rb'

data/config/metrics/saikuro.yml

@@ -0,0 +1,3 @@
+---
+warn_cyclo:  3
+error_cyclo: 5

data/config/metrics/simplecov.yml

@@ -0,0 +1,6 @@
+---
+output: tmp/coverage
+filters: # The list of paths to be excluded from coverage checkup
+- "spec/"
+- "config/"
+groups: []

data/config/metrics/yardstick.yml

@@ -0,0 +1,37 @@
+---
+# Settings added by the 'hexx-suit' gem
+output: "tmp/yardstick/output.log"
+path: "lib/**/*.rb"
+rules:
+  ApiTag::Presence:
+    enabled: true
+    exclude: []
+  ApiTag::Inclusion:
+    enabled: true
+    exclude: []
+  ApiTag::ProtectedMethod:
+    enabled: true
+    exclude: []
+  ApiTag::PrivateMethod:
+    enabled: false
+    exclude: []
+  ExampleTag:
+    enabled: true
+    exclude: []
+  ReturnTag:
+    enabled: true
+    exclude: []
+  Summary::Presence:
+    enabled: true
+    exclude: []
+  Summary::Length:
+    enabled: true
+    exclude: []
+  Summary::Delimiter:
+    enabled: true
+    exclude: []
+  Summary::SingleLine:
+    enabled: true
+    exclude: []
+threshold: 100
+verbose: false

data/lib/rom-kafka.rb

@@ -0,0 +1,3 @@
+# encoding: utf-8
+
+require_relative "rom/kafka"

data/lib/rom/kafka.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+require "attributes_dsl"
+require "poseidon"
+require "rom"
+
+# Ruby Object Mapper
+#
+# @see http://rom-rb.org/
+#
+module ROM
+
+  # Apache Kafka support for ROM
+  #
+  # @see http://kafka.apache.org/
+  #
+  module Kafka
+
+    require_relative "kafka/brokers"
+    require_relative "kafka/connection"
+    require_relative "kafka/dataset"
+    require_relative "kafka/gateway"
+    require_relative "kafka/relation"
+    require_relative "kafka/create"
+
+  end # module Kafka
+
+  register_adapter(:kafka, Kafka)
+
+end # module ROM

data/lib/rom/kafka/brokers.rb

@@ -0,0 +1,72 @@
+# encoding: utf-8
+
+require_relative "brokers/broker"
+
+module ROM::Kafka
+
+  # Value object describing a collection of brokers (host:port)
+  #
+  # Knows how to extract brokers from address lines and options
+  #
+  # @example
+  #   brokers = Brokers.new(
+  #     "localhost:9092",
+  #     "127.0.0.1",
+  #     hosts: ["127.0.0.2:9094"],
+  #     port: 9093,
+  #     unknown_key: :foo # will be ignored by the initializer
+  #   )
+  #
+  #   brokers.to_a
+  #   # => ["localhost:9092", "127.0.0.2:9093", "127.0.0.3:9094"]
+  #
+  # @author Andrew Kozin <Andrew.Kozin@gmail.com>
+  #
+  class Brokers
+
+    include Equalizer.new(:to_a)
+
+    # @!method initialize(lines, options)
+    # Initializes an immutable collection from address lines and/or options
+    #
+    # The initializer is options-tolerant: it just ignores unknown options.
+    #
+    # @param [#to_s, Array<#to_s>] lines
+    # @param [Hash] options
+    #
+    # @option options [#to_s, Array<#to_s>] :hosts
+    # @option options [#to_i] :port
+    #
+    def initialize(*lines)
+      hosts, port = extract_hosts_and_port(lines)
+      @brokers    = extract_brokers(hosts, port)
+
+      IceNine.deep_freeze(self)
+    end
+
+    # Returns array of string representations of brokers
+    #
+    # @return [Array<String>]
+    #
+    def to_a
+      @brokers.map(&:to_s)
+    end
+
+    private
+
+    def extract_hosts_and_port(lines)
+      options = lines.last.instance_of?(Hash) ? lines.pop : {}
+      port    = options[:port]
+      hosts   = (lines + Array[options[:hosts]]).compact.flatten
+
+      [hosts, port]
+    end
+
+    def extract_brokers(hosts, port)
+      brokers = hosts.map { |host| Broker.new(host: host, port: port) }
+      brokers.any? ? brokers : [Broker.new]
+    end
+
+  end # class Brokers
+
+end # module ROM::Kafka

data/lib/rom/kafka/brokers/broker.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+
+module ROM::Kafka
+
+  class Brokers
+
+    # Describes an address to a brocker
+    #
+    # @example
+    #   broker = Broker.new host: "localhost:9092"
+    #   broker.to_s # => "localhost:9092"
+    #
+    #   broker = Broker.new host: "localhost", port: 9092
+    #   broker.to_s # => "localhost:9092"
+    #
+    #   broker = Broker.new host: "localhost:9092", port: 9093
+    #   broker.to_s # => "localhost:9092"
+    #
+    # @author Andrew Kozin <Andrew.Kozin@gmail.com>
+    #
+    class Broker
+
+      include Equalizer.new(:port, :host)
+
+      # Regex to extract host from address line
+      HOST = %r{^\w+(\:\/\/)?\S+(?=\:)|\S+}.freeze
+
+      # Regex to extract port from address line
+      PORT = /(?!\:)\d{4,5}$/.freeze
+
+      # @!attribute [r] host
+      #
+      # @return [String] the host of the broker
+      #
+      attr_reader :host
+
+      # @!attribute [r] port
+      #
+      # @return [Integer] the port of the broker
+      #
+      attr_reader :port
+
+      # Initializes a value object from host line and port
+      #
+      # @option options [#to_s] :host ("localhost")
+      # @option options [#to_i] :port (9092)
+      #
+      def initialize(options = {})
+        line  = options.fetch(:host) { "localhost" }
+        @host = line[HOST]
+        @port = (line[PORT] || options.fetch(:port) { 9092 }).to_i
+
+        IceNine.deep_freeze(self)
+      end
+
+      # Returns the string representation of the broker in "host:port" format
+      #
+      # @return [String]
+      #
+      def to_s
+        "#{host}:#{port}"
+      end
+
+    end # class Broker
+
+  end # class Brokers
+
+end # module ROM::Kafka

data/lib/rom/kafka/connection.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+
+module ROM::Kafka
+
+  # Describes the connection to Kafka cluster
+  #
+  # This is a base abstract class for producer and concumer connections.
+  #
+  # @api private
+  #
+  # @author Andrew Kozin <Andrew.Kozin@gmail.com>
+  #
+  class Connection
+
+    extend AttributesDSL
+
+    require_relative "connection/producer"
+    require_relative "connection/consumer"
+
+  end # class Connection
+
+end # module ROM::Kafka

data/lib/rom/kafka/connection/consumer.rb

@@ -0,0 +1,105 @@
+# encoding: utf-8
+
+module ROM::Kafka
+
+  class Connection
+
+    # The consumer-specific connection to Kafka cluster
+    #
+    # It is wrapped around `Poseidon::Consumer` driver, and responsible for
+    # adopting poseidon API to ROM::Dataset via [#initializer] and [#each]
+    # methods.
+    #
+    # ROM::Kafka consumer deals with tuples, hiding poseidon-specific
+    # implementation of fetched messages from the rest of the gem.
+    #
+    # @api private
+    #
+    class Consumer < Connection
+
+      include Enumerable
+
+      # The 'poseidon'-specific class describing consumers
+      #
+      # @return [Class]
+      #
+      DRIVER = Poseidon::PartitionConsumer
+
+      # Attributes acceptable by the `Poseidon::Consumer` driver
+      attribute :min_bytes
+      attribute :max_bytes
+      attribute :max_wait_ms
+
+      # @!attribute [r] connection
+      #
+      # @return [ROM::Kafka::Connection::Consumer::DRIVER] driver to Kafka
+      #
+      attr_reader :connection
+
+      # Initializes a consumer connection
+      #
+      # The initializer is attributes-agnostic. This means it doesn't validate
+      # attributes, but skips unused.
+      #
+      # @option opts [#to_s] :client_id
+      #   A required unique id used to indentify the Kafka client.
+      # @option opts [Array<String>] :brokers
+      #   A list of seed brokers to find a lead broker to fetch messages from.
+      # @option opts [String] :topic
+      #   A name of the topic to fetch messages from.
+      # @option opts [Integer] :partition
+      #   A number of partition to fetch messages from.
+      # @option opts [Integer] :offset
+      #   An initial offset to start fetching from.
+      # @option opts [Integer] :min_bytes (1)
+      #   A smallest amount of data the server should send.
+      #   (By default send us data as soon as it is ready).
+      # @option opts [Integer] :max_bytes (1_048_576)
+      #   A maximum number of bytes to fetch by consumer (1MB by default).
+      # @option opts [Integer] :max_wait_ms (100)
+      #   How long to block until the server sends data.
+      #   NOTE: This is only enforced if min_bytes is > 0.
+      #
+      def initialize(opts)
+        super # takes declared attributes from options
+        args = opts.values_at(:client_id, :brokers, :topic, :partition, :offset)
+        @connection = DRIVER.consumer_for_partition(*args, attributes)
+      end
+
+      # Fetches a single portion of messages and converts them to tuple
+      #
+      # @return [Array<Hash{Symbol => String, Integer}>]
+      #
+      def fetch
+        result = @connection.fetch
+        result.map(&method(:tuple))
+      end
+
+      # Iterates through Kafka messages
+      #
+      # Fetches the next portion of messages until no messages given
+      #
+      # @return [Enumerator<Array<Hash{Symbol => String, Integer}>>]
+      #
+      # @yieldparam [Hash{Symbol => String, Integer}] tuple
+      #
+      def each
+        return to_enum unless block_given?
+        loop do
+          tuples = fetch
+          break unless tuples.any?
+          tuples.each { |tuple| yield(tuple) }
+        end
+      end
+
+      private
+
+      def tuple(msg)
+        { value: msg.value, topic: msg.topic, key: msg.key, offset: msg.offset }
+      end
+
+    end # class Consumer
+
+  end # class Connection
+
+end # module ROM::Kafka