avro2kafka 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 442cbcdebce5cdbf488aff50c61ab23dafbece43
-  data.tar.gz: 46a629b705531af0ada32f54987af81814772fd8
+  metadata.gz: d6e62da76fc477504898300a2a07e30d1c87d09e
+  data.tar.gz: a582a13d5231bfa9f2b56bc6033d706193573b08
 SHA512:
-  metadata.gz: 6e9ec180f4fbe39e175fce8caa2dd97180f99faf35dfce6a50bde5b4fd4775380978b937602c57d6c3cb3cfe68fb301568de19a3c543eecb9d4d7b11b169531c
-  data.tar.gz: b4ee9b33830dbfbd22b9ee8b7064263c9df79d8b149968353e69c8a70b7035086ca19661d447a5dbd5d161b3fedd7c026af88e0454b838b9611cbef822680a68
+  metadata.gz: c6d08de11aca94e79abb24a179f752ea574ef772d7eea54a780cfa08f11ef9ba681431f4b932a1d72acc1dc110094f19586695f6d4446880dd45c8f76072e4a6
+  data.tar.gz: 6aded16c6bb9b00b215723629d5d10435cfc49f9277ad87dded43616099401b8bc5699b08173d7fa3bf3ca927d8a016291313aa21ca2d6db0949baecace21d59

data/.rspec

@@ -0,0 +1 @@
+--color

data/CHANGELOG.md

@@ -3,12 +3,16 @@
 All notable changes to this project are documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
-## [Unreleased]
+## 0.2.0 (2015-06-12)
 
 ### Added
 
+  * --data option for extra data to be added to the message payload
+
 ### Changed
 
+  * option --broker renamed to --broker-list and accpeting a comma separated list of brokers
+
 ## 0.1.0 (2015-06-09)
 Initial release
 

data/README.md

@@ -26,14 +26,20 @@ Usage: avro2kafka [options] [file]
 
 ### Options
 
-`-b, --broker BROKER`
-The Kafka broker, eg. localhost:9092. This value is required.
+`-b, --broker-list BROKER`
+A comma-separated list of Kafka brokers, eg. localhost:9092. This value is required.
 
 `-t, --topic TOPIC`
 The Kafka topic to publish to. This value is required.
 
 `-k, --key KEY`
-The fields in the avro that will act as keys for Kafka. If not supplied, unique and undefined keys will be generated, hence log compaction won't work.
+A comma-separated list of fields in the avro that will act as keys for Kafka. If not supplied, unique and undefined keys will be generated, hence log compaction won't work.
+
+`-d, --data DATA`
+Extra data to pass along with the message payload. Format: key=value
+Multiple arguments can be defined. For example:
+
+    $ avro2kafka [...] -d pwd=$PWD -d site=acme
 
 `-h, --help`
 Prints help

data/bin/avro2kafka

@@ -10,8 +10,8 @@ option_parser = OptionParser.new do |opts|
   opts.banner = "Version #{Avro2Kafka::VERSION} of Avro2Kafka\n" \
     "Usage: #{File.basename(__FILE__)} [options] [file]"
 
-  opts.on('-b', '--broker BROKER', 'The Kafka broker, eg. localhost:9092. This value is required.') do |broker|
-    options[:broker] = broker
+  opts.on('-b', '--broker-list BROKER', 'The Kafka broker list. Format: host:port. This value is required.') do |broker_list|
+    options[:broker_list] = broker_list
   end
 
   opts.on('-t', '--topic TOPIC', 'The Kafka topic to publish to. This value is required.') do |topic|
@@ -22,6 +22,11 @@ option_parser = OptionParser.new do |opts|
     options[:key] = key
   end
 
+  opts.on('-d', '--data DATA', 'Extra data fields to add to the kafka payload. Format: key=value') do |data|
+    options[:data] ||= []
+    options[:data] << data
+  end
+
   opts.on('-h', '--help', 'Prints help') do
     puts opts
     exit
@@ -31,7 +36,7 @@ end
 option_parser.parse!
 
 begin
-  raise OptionParser::MissingArgument.new('--broker') if options[:broker].nil?
+  raise OptionParser::MissingArgument.new('--broker-list') if options[:broker_list].nil?
   raise OptionParser::MissingArgument.new('--topic') if options[:topic].nil?
 
   Avro2Kafka.new(options).publish

data/lib/avro2kafka.rb

@@ -3,26 +3,44 @@ require 'avro2kafka/avro_reader'
 require 'avro2kafka/kafka_publisher'
 
 class Avro2Kafka
-  attr_reader :input_path, :kafka_broker, :kafka_topic, :kafka_key
+  attr_reader :options
 
   def initialize(options)
-    @input_path = ARGV.first
-    @kafka_broker = options.delete(:broker)
-    @kafka_topic = options.delete(:topic)
-    @kafka_key = options[:key]
-
     @options = options
   end
 
   def reader
-    ARGF.lineno = 0
-    ARGF
+    ARGF.tap { |argf| argf.rewind }
   end
 
   def publish
     records = AvroReader.new(reader).read
-    KafkaPublisher.new(kafka_broker, kafka_topic, kafka_key).publish(records)
-    puts "Avro file published to #{kafka_topic} topic on #{kafka_broker}!"
+    KafkaPublisher.new(**kafka_options).publish(records)
+    puts "Avro file published to #{topic} topic on #{broker_list}!"
+  end
+
+  def topic
+    options.fetch(:topic)
+  end
+
+  def broker_list
+    options.fetch(:broker_list).split(',')
+  end
+
+  def extra_data
+    options.fetch(:data, []).each_with_object({}) do |data, hash|
+      key, value = data.split('=')
+      hash[key] = value
+    end
+  end
+
+  def kafka_options
+    {
+      broker_list: broker_list,
+      topic: topic,
+      data: extra_data,
+      keys: options.fetch(:key, '').split(',').map(&:strip),
+    }
   end
 
 end

data/lib/avro2kafka/kafka_publisher.rb

@@ -3,26 +3,32 @@ require 'json'
 
 class Avro2Kafka
   class KafkaPublisher
-    attr_reader :producer, :topic, :keys
+    attr_reader :producer, :topic, :keys, :data
 
-    def initialize(broker, topic, keys)
-      @producer = Poseidon::Producer.new([broker], "avro2kafka")
+    def initialize(broker_list:, topic:, keys: [], data: {})
+      @producer = Poseidon::Producer.new(broker_list, 'avro2kafka')
       @topic = topic
-      @keys = keys.split(',').map(&:strip) if keys
+      @keys = keys
+      @data = data
     end
 
     def publish(records)
       records.each_slice(100) do |batch|
-        messages = batch.map do |record|
-          if keys
-            message_key = keys.map { |key| record[key] }.join
-            Poseidon::MessageToSend.new(topic, record.to_json, message_key)
-          else
-            Poseidon::MessageToSend.new(topic, record.to_json)
-          end
-        end
+        messages = batch.map { |record| prepare_record(record) }
         producer.send_messages(messages)
       end
     end
+
+    private
+
+    def prepare_record(record)
+      record = record.merge(data) if record.is_a?(Hash)
+      if keys.empty?
+        Poseidon::MessageToSend.new(topic, record.to_json)
+      else
+        message_key = keys.map { |key| record[key] }.join
+        Poseidon::MessageToSend.new(topic, record.to_json, message_key)
+      end
+    end
   end
 end

data/lib/avro2kafka/version.rb

@@ -1,3 +1,3 @@
 class Avro2Kafka
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/spec/avro2kafka/kafka_publisher_spec.rb

@@ -3,17 +3,37 @@ require 'poseidon'
 
 RSpec.describe Avro2Kafka::KafkaPublisher do
   describe '#publish' do
-    let(:broker) { 'localhost:9092' }
+    let(:broker_list) { %w[localhost:9092] }
     let(:topic) { 'feeds' }
-    let(:key) { 'name, id' }
-    let(:messages) { %w[message1, message2, message3] }
+    let(:keys) { %w[name id] }
+    let(:data) { {'site' => 'acme' } }
+    let(:messages) { [{ 'id' => 1, 'name' => 'dresses' }, { 'id' => 2, 'name' => 'tops' }] }
+    let(:publisher) { Avro2Kafka::KafkaPublisher.new(broker_list: broker_list, topic: topic, keys: keys, data: data) }
+    let(:producer) { double('producer') }
+
+    before do
+      allow(publisher).to receive(:producer).and_return(producer)
+    end
 
     it 'should call publisher#send_messages' do
-      kafka_publisher = Avro2Kafka::KafkaPublisher.new(broker, topic, key)
-      mock_producer = double("Producer")
-      allow(kafka_publisher).to receive(:producer).and_return(mock_producer)
-      expect(mock_producer).to receive(:send_messages).with(all(be_kind_of(Poseidon::MessageToSend)))
-      kafka_publisher.publish(messages)
+      expect(producer).to receive(:send_messages).with(all(be_kind_of(Poseidon::MessageToSend)))
+      publisher.publish(messages)
+    end
+
+    it 'should create keys' do
+      expect(producer).to receive(:send_messages) do |args|
+        expect(args).to all(have_attributes(key: a_value))
+      end
+
+      publisher.publish(messages)
+    end
+
+    it 'should add extra data to payload' do
+      expect(producer).to receive(:send_messages) do |args|
+        expect(args.map { |a| JSON.load(a.value) }).to all(include({ 'site' => 'acme' }))
+      end
+
+      publisher.publish(messages)
     end
   end
 end

data/spec/avro2kafka_spec.rb

@@ -6,15 +6,58 @@ RSpec.describe Avro2Kafka do
   describe '#publish' do
     let(:options) do
       {
-        broker: 'localhost:9092',
+        broker_list: 'localhost:9092,localhost:9093',
         topic: 'feeds',
-        key: 'name, id'
+        key: 'name,id',
+        data: %w[x=y alpha=omega]
       }
     end
+    let(:avro2kafka) { Avro2Kafka.new(options) }
 
     before do
       ARGV.replace ['./spec/support/data.avro']
     end
 
+    describe '#topic' do
+      it 'returns the topic' do
+        expect(avro2kafka.topic).to eq 'feeds'
+      end
+
+      it 'throws an error if topic is not present' do
+        options.delete(:topic)
+        expect { avro2kafka.topic }.to raise_error
+      end
+    end
+
+    describe '#broker_list' do
+      it 'splits the broker list by commas' do
+        expect(avro2kafka.broker_list).to eq %w[localhost:9092 localhost:9093]
+      end
+
+      it 'throws an error if broker_list is not present' do
+        options.delete(:broker_list)
+        expect { avro2kafka.broker_list }.to raise_error
+      end
+    end
+
+    describe '#extra_data' do
+      it 'parses data into key-value pairs' do
+        expect(avro2kafka.extra_data).to eq({ 'x' => 'y', 'alpha' => 'omega' })
+      end
+    end
+
+    describe '#publish' do
+      around do |ex|
+        stdout_old = $stdout
+        $stdout = File.open(File::NULL, 'w')
+        ex.run
+        $stdout = stdout_old
+      end
+
+      it 'forwards the records to KafkaPublisher#publish' do
+        expect_any_instance_of(Avro2Kafka::KafkaPublisher).to receive(:publish)
+        avro2kafka.publish
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: avro2kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Peter Marton
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-11 00:00:00.000000000 Z
+date: 2015-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -116,6 +116,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".rspec"
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock