rom-kafka 0.0.1

data/lib/rom/kafka/relation.rb

@@ -0,0 +1,78 @@
+# encoding: utf-8
+
+module ROM::Kafka
+
+  # The Kafka-specific implementation of ROM::Relation
+  #
+  # @example
+  #   ROM.use(:auto_registration)
+  #   ROM.setup(:kafka, "localhost:9092")
+  #
+  #   class Users < ROM::Relation[:kafka]
+  #     topic "users"
+  #   end
+  #
+  #   rom = ROM.finalize.env
+  #   users = rom.relation(:users)
+  #   users.where(partition: 1).offset(0).limit(1).to_a
+  #   # => [
+  #   #      { value: "Andrew", topic: "users", partition: 1, offset: 0 }
+  #   #    ]
+  #
+  class Relation < ROM::Relation
+
+    adapter :kafka
+
+    # Kafka-specific alias for the ROM `.dataset` helper method.
+    #
+    # @param [#to_sym] name
+    #
+    # @return [undefined]
+    #
+    def self.topic(name)
+      dataset(name)
+    end
+
+    # Returns new relation with updated `:partition` attribute
+    #
+    # @param [Integer] value
+    #
+    # @return [ROM::Kafka::Relation]
+    #
+    def from(value)
+      using(partition: value)
+    end
+
+    # Returns new relation with updated `:offset` attribute
+    #
+    # @param [Integer] value
+    #
+    # @return [ROM::Kafka::Relation]
+    #
+    def offset(value)
+      using(offset: value)
+    end
+
+    # Returns new relation with updated `:limit` attribute
+    #
+    # @param [Integer] value
+    #
+    # @return [ROM::Kafka::Relation]
+    #
+    def limit(value)
+      using(limit: value)
+    end
+
+    # Returns new relation where dataset is updated with given attributes
+    #
+    # @param [Hash] attributes
+    #
+    # @return [ROM::Kafka::Relation]
+    #
+    def using(attributes)
+      self.class.new dataset.using(attributes)
+    end
+
+  end # class Relation
+
+end # module ROM::Kafka

data/lib/rom/kafka/version.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+
+module ROM
+
+  module Kafka
+
+    # The semantic version of the module.
+    # @see http://semver.org/ Semantic versioning 2.0
+    VERSION = "0.0.1".freeze
+
+  end # module Kafka
+
+end # module ROM

data/rom-kafka.gemspec

@@ -0,0 +1,33 @@
+# encoding: utf-8
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "rom/kafka/version"
+
+Gem::Specification.new do |gem|
+
+  gem.name             = "rom-kafka"
+  gem.version          = ROM::Kafka::VERSION.dup
+  gem.author           = ["Andrew Kozin"]
+  gem.email            = ["andrew.kozin@gmail.com"]
+  gem.summary          = "Kafka support for Ruby Object Mapper"
+  gem.description      = gem.summary
+  gem.homepage         = "https://rom-rb.org"
+  gem.license          = "MIT"
+
+  gem.files            = `git ls-files -z`.split("\x0")
+  gem.executables      = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  gem.test_files       = gem.files.grep(%r{^(test|spec|features)/})
+  gem.extra_rdoc_files = Dir["README.md", "LICENSE"]
+  gem.require_paths    = ["lib"]
+
+  gem.required_ruby_version = "~> 1.9", ">= 1.9.3"
+
+  gem.add_runtime_dependency "rom", "~> 0.9", ">= 0.9.1"
+  gem.add_runtime_dependency "poseidon", "~> 0.0", ">= 0.0.5"
+  gem.add_runtime_dependency "attributes_dsl", "~> 0.0", ">= 0.0.2"
+
+  gem.add_development_dependency "hexx-rspec", "~> 0.5"
+  gem.add_development_dependency "inflecto", "~> 0.0", ">= 0.0.2"
+  gem.add_development_dependency "timecop", "~> 0.8"
+
+end # Gem::Specification

data/spec/integration/basic_usage_spec.rb

@@ -0,0 +1,58 @@
+# encoding: utf-8
+require "shared/scholars_topic"
+
+describe "Basic Usage" do
+
+  include_context :scholars_topic
+
+  let(:add_scholars) { insert.with(key: 0) }
+
+  it "works" do
+    # Add messages into the 0 partition (see :add_scholars above)
+    expect(add_scholars.call("Matthew", "Mark").to_a).to eql [
+      { value: "Matthew", topic: "scholars", key: "0" },
+      { value: "Mark",    topic: "scholars", key: "0" }
+    ]
+
+    # Fetching from the <default> 0 partition gives all messages
+    expect(scholars.call.to_a).to eql [
+      { value: "Matthew", topic: "scholars", key: "0", offset: 0 },
+      { value: "Mark",    topic: "scholars", key: "0", offset: 1 }
+    ]
+
+    # Second call returns nothing because all messages were already fetched
+    expect(scholars.call.to_a).to eql []
+
+    # Add a couple of messages
+    expect(add_scholars.call("Luke", "John").to_a).to eql [
+      { value: "Luke", topic: "scholars", key: "0" },
+      { value: "John", topic: "scholars", key: "0" }
+    ]
+
+    # And fetch them (now starting from the next offset from we stay before)
+    expect(scholars.call.to_a).to eql [
+      { value: "Luke", topic: "scholars", key: "0", offset: 2 },
+      { value: "John", topic: "scholars", key: "0", offset: 3 }
+    ]
+
+    # Re-fetch all the messages from 0 offset
+    expect(scholars.offset(0).call.to_a).to eql [
+      { value: "Matthew", topic: "scholars", key: "0", offset: 0 },
+      { value: "Mark",    topic: "scholars", key: "0", offset: 1 },
+      { value: "Luke",    topic: "scholars", key: "0", offset: 2 },
+      { value: "John",    topic: "scholars", key: "0", offset: 3 }
+    ]
+
+    # Re-fetch only limited subset of messages
+    expect(scholars.offset(1).limit(2).call.to_a).to eql [
+      { value: "Mark", topic: "scholars", key: "0", offset: 1 },
+      { value: "Luke", topic: "scholars", key: "0", offset: 2 }
+    ]
+
+    # But actually this will move the offset not to 2 but to the end
+    # (consumer fetched all the messages, but iterated via 2 only)
+    # To start from the next offset, we should set it explicitly.
+    expect(scholars.call.to_a).to eql []
+  end
+
+end # describe Basic Usage

data/spec/integration/keys_usage_spec.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+require "shared/scholars_topic"
+
+describe "Keys Usage" do
+
+  include_context :scholars_topic
+
+  it "works" do
+    # Add messages into the partition 1, to be extracted from key "4"
+    expect(insert.with(key: 4).call("Thomas", "Judah").to_a).to eql [
+      { value: "Thomas", topic: "scholars", key: "4" },
+      { value: "Judah",  topic: "scholars", key: "4" }
+    ]
+
+    # Add messages into the partition 2, to be extracted from key "5"
+    expect(insert.with(key: 5).call("Maria", "Philip").to_a).to eql [
+      { value: "Maria",  topic: "scholars", key: "5" },
+      { value: "Philip", topic: "scholars", key: "5" }
+    ]
+
+    # Look at the data in the partition 1
+    expect(scholars.from(1).offset(0).call.to_a).to eql [
+      { value: "Thomas", topic: "scholars", key: "4", offset: 0 },
+      { value: "Judah",  topic: "scholars", key: "4", offset: 1 }
+    ]
+
+    # Look at the data in the partition 2
+    expect(scholars.from(2).offset(0).call.to_a).to eql [
+      { value: "Maria",  topic: "scholars", key: "5", offset: 0 },
+      { value: "Philip", topic: "scholars", key: "5", offset: 1 }
+    ]
+  end
+
+end # describe Keys Usage

data/spec/shared/scholars_topic.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+shared_examples :scholars_topic do
+
+  let!(:rom) do
+    env = ROM::Environment.new
+    env.use :auto_registration
+
+    setup = env.setup(
+      :kafka, "localhost:9092",
+      client_id: "admin",
+      # use the number of partition as a key
+      partitioner: -> key, total { key.to_i % total }
+    )
+
+    setup.relation(:scholars)
+    setup.commands(:scholars) do
+      define(:create)
+    end
+
+    setup.finalize
+    setup.env
+  end
+
+  let(:scholars) { rom.relation(:scholars) }
+  let(:insert)   { rom.command(:scholars).create }
+
+end # shared_examples

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+
+begin
+  require "hexx-suit"
+  Hexx::Suit.load_metrics_for(self)
+rescue LoadError
+  require "hexx-rspec"
+  Hexx::RSpec.load_metrics_for(self)
+end
+
+# Loads the code under test
+require "rom-kafka"
+
+# @todo Remove after resolving of mutant PR#444
+# @see https://github.com/mbj/mutant/issues/444
+if ENV["MUTANT"]
+  RSpec.configure do |config|
+    config.around { |example| Timeout.timeout(0.5, &example) }
+  end
+end

data/spec/unit/brokers/broker_spec.rb

@@ -0,0 +1,89 @@
+# encoding: utf-8
+
+describe ROM::Kafka::Brokers::Broker do
+
+  let(:broker) { described_class.new }
+
+  describe ".new" do
+    subject { broker }
+
+    it { is_expected.to be_frozen }
+  end # describe .new
+
+  describe "#host" do
+    subject { broker.host }
+
+    context "by default" do
+      it { is_expected.to eql "localhost" }
+    end
+
+    context "when full :host is given" do
+      let(:broker) { described_class.new host: :"https://some.path.com:9093" }
+
+      it { is_expected.to eql "https://some.path.com" }
+    end
+
+    context "when localhost is given" do
+      let(:broker) { described_class.new host: :"localhost:9093" }
+
+      it { is_expected.to eql "localhost" }
+    end
+  end # describe #host
+
+  describe "#port" do
+    subject { broker.port }
+
+    context "by default" do
+      it { is_expected.to eql 9092 }
+    end
+
+    context "when :host contains port" do
+      let(:broker) { described_class.new host: :"https://some.path.com:9093" }
+
+      it { is_expected.to eql 9093 }
+    end
+
+    context "when :port is set" do
+      let(:broker) { described_class.new port: 9093 }
+
+      it { is_expected.to eql 9093 }
+    end
+
+    context "when :host contains port and :port is set" do
+      let(:broker) { described_class.new host: "localhost:9092", port: 9093 }
+
+      it "prefers the host setting" do
+        expect(subject).to eql 9092
+      end
+    end
+  end # describe #port
+
+  describe "#to_s" do
+    subject { described_class.new(host: :"127.0.0.1:9093").to_s }
+
+    it { is_expected.to eql "127.0.0.1:9093" }
+  end # describe #to_s
+
+  describe "#==" do
+    subject { broker == other }
+
+    context "with the same host and port" do
+      let(:other) { described_class.new }
+
+      it { is_expected.to eql true }
+    end
+
+    context "with another host" do
+      let(:other) { described_class.new host: "127.0.0.1" }
+
+      it { is_expected.to eql false }
+    end
+
+    context "with another port" do
+      let(:other) { described_class.new port: 9093 }
+
+      it { is_expected.to eql false }
+    end
+  end # describe #==
+
+end # describe ROM::Kafka::Brokers::Broker

data/spec/unit/brokers_spec.rb

@@ -0,0 +1,46 @@
+# encoding: utf-8
+
+describe ROM::Kafka::Brokers do
+
+  let(:default_brokers) { described_class.new }
+  let(:custom_brokers) do
+    described_class.new "foo", "bar:9093", hosts: ["baz:9092"], port: 9094
+  end
+
+  describe ".new" do
+    subject { default_brokers }
+
+    it { is_expected.to be_frozen }
+  end # describe .new
+
+  describe "#to_a" do
+    context "by default" do
+      subject { default_brokers.to_a }
+
+      it { is_expected.to eql ["localhost:9092"] }
+    end
+
+    context "customized" do
+      subject { custom_brokers.to_a }
+
+      it { is_expected.to eql ["foo:9094", "bar:9093", "baz:9092"] }
+    end
+  end # describe #to_a
+
+  describe "#==" do
+    subject { default_brokers == other }
+
+    context "with the same brokers" do
+      let(:other) { described_class.new }
+
+      it { is_expected.to eql true }
+    end
+
+    context "with different brokers" do
+      let(:other) { described_class.new "foo" }
+
+      it { is_expected.to eql false }
+    end
+  end # describe #==
+
+end # describe ROM::Kafka::Brokers

data/spec/unit/connection/consumer_spec.rb

@@ -0,0 +1,90 @@
+# encoding: utf-8
+
+describe ROM::Kafka::Connection::Consumer do
+
+  # ============================================================================
+  # We test not the poseidon API, but its proper usage by the Consumer.
+  # That's why we stub poseidon classes.
+  # ----------------------------------------------------------------------------
+  let(:driver)     { Poseidon::PartitionConsumer }
+  let(:connection) { double :connection }
+  before { allow(driver).to receive(:consumer_for_partition) { connection } }
+  # ============================================================================
+
+  let(:consumer) { described_class.new options }
+  let(:options) do
+    attributes.merge(
+      client_id: client,
+      brokers: brokers,
+      topic: topic,
+      partition: partition,
+      offset: offset
+    )
+  end
+  let(:attributes) { { min_bytes: 2, max_bytes: 3000, max_wait_ms: 100 } }
+  let(:brokers)    { ["127.0.0.1:9092", "127.0.0.2:9092"] }
+  let(:client)     { "foo" }
+  let(:topic)      { "bar" }
+  let(:partition)  { 1 }
+  let(:offset)     { 100 }
+  let(:tuple)      { { value: "Hi!", topic: "foo", key: "foo", offset: 100 } }
+  let(:message)    { double :message, tuple }
+
+  describe ".new" do
+    subject { consumer }
+
+    it { is_expected.to be_kind_of Enumerable }
+  end # describe .new
+
+  describe "#connection" do
+    subject { consumer.connection }
+
+    it "instantiates the driver" do
+      expect(driver)
+        .to receive(:consumer_for_partition)
+        .with(client, brokers, topic, partition, offset, attributes)
+
+      expect(subject).to eql(connection)
+    end
+  end # describe #connection
+
+  describe "#fetch" do
+    subject { consumer.fetch }
+
+    before  { allow(connection).to receive(:fetch) { [message] } }
+
+    it "fetches messages from a connection" do
+      expect(connection).to receive(:fetch)
+      expect(subject).to eql [tuple]
+    end
+  end # describe #fetch
+
+  describe "#each" do
+
+    let(:messages) { [message, message] } # stub messages to extract from broker
+    before do
+      allow(connection)
+        .to receive(:fetch) do
+          data = [messages.pop].compact
+          messages.freeze unless data.any? # the next `pop` should fail
+          data
+        end
+    end
+
+    context "without a block" do
+      subject { consumer.each }
+
+      it { is_expected.to be_kind_of Enumerator }
+    end
+
+    context "with a block" do
+      subject { consumer.to_a }
+
+      it "fetches messages while received any" do
+        expect(connection).to receive(:fetch).exactly(3).times
+        expect(subject).to eq [tuple, tuple]
+      end
+    end
+  end # describe #each
+
+end # describe ROM::Kafka::Connection::Consumer