kril 0.1.0

data/.travis.yml

@@ -0,0 +1,22 @@
+language: ruby
+cache: bundler
+
+rvm:
+  - 2.4.0
+
+before_install: gem install bundler -v 1.16.0
+
+notifications:
+  email:
+    recipients:
+      - chad.bowman0@gmail.com
+    on_failure: change
+    on_success: never
+
+deploy:
+  provider: pages
+  skip_cleanup: true
+  github_token: $GITHUB_TOKEN
+  project_name: kril
+  on:
+    branch: master

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Chad Bowman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,89 @@
+# Kril 🦐
+
+Kril is an easy to use command line interface (CLI) for interacting with [Apache Kafka](https://kafka.apache.org/). It uses [Apache Avro](https://avro.apache.org/) for serialization/deserialization.
+
+[![Build Status](https://travis-ci.org/ChadBowman/kril.svg?branch=master)](https://travis-ci.org/ChadBowman/kril)
+
+## Installation
+
+Add this line to your application's Gemspec:
+
+```ruby
+spec.add_development_dependency 'kril', '~> 0.1'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install kril
+
+## Usage
+
+Assuming your schema is not registered with the schema registry, and doesn't exist in the repository, you can define a schema and commit a record like so:
+```bash
+$ kril --bootstrap-servers 'localhost:9092,localhost:9093,localhost:9094' \
+--schema-registry 'http://localhost:8081' \
+--with-schema '{"type":"record","name":"human","fields":[{"name":"age","type":"int"}]}' \
+--record '{"age": 27}' \
+human
+```
+```bash
+🦐 saved human: {"type"=>"record", "name"=>"human", "fields"=>[{"name"=>"age", "type"=>"int"}]}
+🦐 human: {"age"=>27}
+```
+
+Now we can consume a single record:
+```bash
+$ kril --pretty-print human
+```
+```bash
+🦐 human: 
+{
+  "key": null,
+  "value": {
+    "age": 27
+  },
+  "offset": 0,
+  "create_time": "2018-03-04 00:29:47 -0700",
+  "topic": "human",
+  "partition": 4
+}
+```
+---
+Now that the schema exists, we can produce records simply:
+```bash
+$ kril -r '{"age": 33}' human
+```
+```bash
+🦐 human: {"age"=>33}
+```
+---
+Consuming all records ever:
+```bash
+$ kril --consume-all human
+```
+```bash
+🦐 human: {:key=>nil, :value=>{"age"=>27}, :offset=>0, :create_time=>2018-03-04 00:12:32 -0700, :topic=>"human", :partition=>2}
+🦐 human: {:key=>nil, :value=>{"age"=>27}, :offset=>0, :create_time=>2018-03-04 00:29:47 -0700, :topic=>"human", :partition=>4}
+🦐 human: {:key=>nil, :value=>{"age"=>27}, :offset=>0, :create_time=>2018-03-04 00:26:33 -0700, :topic=>"human", :partition=>1}
+🦐 human: {:key=>nil, :value=>{"age"=>27}, :offset=>0, :create_time=>2018-03-04 00:25:54 -0700, :topic=>"human", :partition=>3}
+🦐 human: {:key=>nil, :value=>{"age"=>33}, :offset=>1, :create_time=>2018-03-04 00:34:07 -0700, :topic=>"human", :partition=>3}
+🦐 human: {:key=>nil, :value=>{"age"=>27}, :offset=>0, :create_time=>2018-03-04 00:13:13 -0700, :topic=>"human", :partition=>0}
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/ChadBowman/kril/fork )
+2. Create your feature branch (git checkout -b my-new-feature)
+3. Commit your changes (git commit -am 'add some feature')
+4. Push to the branch (git push origin my-new-feature)
+5. Create a new Pull Request
+
+Please try to obey [Rubocop](https://github.com/bbatsov/rubocop) to the best of your abilities.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rspec/core/rake_task'
+require 'bundler/gem_tasks'
+
+# Default directory to look in is `/specs`
+# Run with `rake spec`
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.rspec_opts = ['--color', '--format documentation']
+end
+
+task :default => :spec

data/bin/kril

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+$VERBOSE = ARGV.map { |arg| true if arg.match?(/-v|--verbose/) }.compact.first
+
+require 'avro_turf/messaging'
+require 'kafka'
+require 'clamp'
+require 'kril'
+
+Clamp do
+  option %w[-k --bootstrap-servers], '', 'address(es) of kafka cluster',
+         default: %w[localhost:9092 localhost:9093 localhost:9094] do |address_string|
+    address_string.split(/,\s*/)
+  end
+  # configuration
+  option %w[-g --schema-registry], '', 'address of schema registry', default: 'http://localhost:8081'
+  option %w[-p --schemas-path], '', 'directory of avro schemas', default: 'schemas/'
+  option %w[-v --verbose], :flag, 'print logs, warnings'
+  option %w[-e --pretty-print], :flag, 'pretty print records', default: false
+
+  # producing
+  option %w[-r --record], '', 'record to commit to topic'
+  option %w[-o --syncronous], :flag, 'commit records syncronously', default: false
+  option %w[-w --with-schema], '', 'add schema to respository'
+
+  # consuming
+  option %w[-f --follow], :flag, 'consume from topic indefinitely', defualt: false
+  option %w[-a --consume-all], :flag, 'consume every record on topic', default: false
+
+  parameter 'TOPIC', 'topic to produce to or consume from'
+  parameter '[NAME]', 'schema name'
+
+  def execute
+    if topic
+      log = Logger.new($STDOUT)
+      log.level = verbose? ? Logger::WARN : Logger::ERROR
+      kafka = Kafka.new(bootstrap_servers, logger: log, client_id: 'kril')
+      avro = AvroTurf::Messaging.new(registry_url: schema_registry,
+                                     schemas_path: schemas_path,
+                                     logger: log)
+
+      if with_schema
+        schema = JSON.parse(with_schema)
+        schema_name = schema['name']
+        path = File.join(schemas_path, "#{schema_name}.avsc")
+        File.open(path, 'w') { |file| file.write(JSON.pretty_generate(schema)) }
+        print_record("saved #{schema_name}", schema)
+      end
+
+      if record
+        producer = Kril::Producer.new(kafka: kafka, avro: avro)
+        rec = JSON.parse(record)
+        producer.send(record: rec,
+                      schema_name: name || topic,
+                      topic: topic,
+                      syncronous: syncronous?)
+        print_record(topic, rec)
+      else
+        consumer = Kril::Consumer.new(kafka: kafka, avro: avro)
+        if consume_all?
+          consumer.consume_all(topic) do |message|
+            print_record(topic, message) 
+          end
+        elsif follow?
+          consumer.listen(topic) do |message|
+            print_record(topic, message)
+          end
+        else
+          print_record(topic, consumer.consume_one(topic))
+        end
+      end
+    end
+  end
+
+  def print_record(topic, record)
+    record = "\n#{JSON.pretty_generate(record)}" if pretty_print?
+    puts "🦐 #{topic}: #{record}"
+  end
+end

data/kril.gemspec

@@ -0,0 +1,35 @@
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kril/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'kril'
+  spec.version       = Kril::VERSION
+  spec.authors       = ['Chad Bowman']
+  spec.email         = ['chad.bowman0@gmail.com']
+
+  spec.summary       = 'A simple command line tool for interacting with Kafka'
+  spec.description   = 'Makes producing and consuming topics simple. Useful when experimenting.'
+  spec.homepage      = 'https://github.com/ChadBowman/kril'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  spec.bindir        = 'bin'
+  spec.executables   = ['kril']
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'avro_turf', '~> 0.8.0'
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'clamp', '~> 1.2', '>= 1.2.1'
+  spec.add_development_dependency 'httparty', '~> 0.16.0'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.7'
+  spec.add_development_dependency 'rspec-nc', '~> 0.3.0'
+  spec.add_development_dependency 'ruby-kafka', '~> 0.5.3'
+  spec.add_development_dependency 'sinatra', '~> 2.0', '>= 2.0.1'
+  spec.add_development_dependency 'webmock', '~> 3.3'
+end

data/lib/kril/consumer.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Kril
+  # Consumers records from Kafka
+  class Consumer
+    def initialize(avro: nil, kafka: nil, config: {})
+      config[:group_id] ||= '🦐'
+      @avro = avro
+      @kafka = kafka
+      @config = config
+    end
+
+    def consume_one(topic)
+      consumer = build_consumer(topic, true, @config)
+      msg = nil
+      consumer.each_message do |message|
+        msg = decode(message)
+        consumer.mark_message_as_processed(message)
+        consumer.commit_offsets
+        consumer.stop
+      end
+      msg
+    ensure
+      consumer.stop
+    end
+
+    def consume_all(topic)
+      config = @config.clone
+      config[:group_id] = SecureRandom.uuid
+      consumer = build_consumer(topic, true, config)
+      consumer.each_message do |message|
+        yield decode(message), consumer
+      end
+    ensure
+      consumer.stop
+    end
+
+    def listen(topic)
+      consumer = build_consumer(topic, false, @config)
+      consumer.each_message do |message|
+        yield decode(message), consumer
+      end
+    ensure
+      consumer.stop
+    end
+
+    private
+
+    def build_consumer(topic, start_from_beginning, config)
+      consumer = @kafka.consumer(config)
+      consumer.subscribe(topic, start_from_beginning: start_from_beginning)
+      consumer
+    end
+
+    def decode(message)
+      {
+        key: message.key,
+        value: @avro.decode(message.value),
+        offset: message.offset,
+        create_time: message.create_time,
+        topic: message.topic,
+        partition: message.partition
+      }
+    end
+  end
+end

data/lib/kril/producer.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Kril
+  # Produces records to Kafka
+  class Producer
+    def initialize(avro: nil, kafka: nil, config: {})
+      config[:required_acks] ||= 1
+      config[:delivery_threshold] ||= 1
+      sync_config = config.dup
+      @avro = avro
+      @async = kafka.async_producer(config)
+      sync_config.delete(:delivery_threshold)
+      @sync = kafka.producer(sync_config)
+    end
+
+    def send(record:, schema_name:, topic: nil, syncronous: false)
+      topic ||= schema_name
+      encoded = @avro.encode(record, schema_name: schema_name)
+      if syncronous
+        @producer.produce(encoded, topic: topic)
+        @producer.deliver_messages
+      else
+        @async.produce(encoded, topic: topic)
+      end
+    ensure
+      @async.shutdown
+      @sync.shutdown
+    end
+  end
+end

data/lib/kril/record_builder.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Kril
+  # Creates and validates a record based from a defined schema.
+  class RecordBuilder
+    def initialize(schema_file_name: nil,
+                   schemas_dir: 'schemas',
+                   schema_path: nil)
+      path = schema_path || File.join(schemas_dir, "#{schema_file_name}.avsc")
+      file = File.read(path)
+      @schema = JSON.parse(file)
+    end
+
+    def build(data)
+      data = JSON.parse(data)
+      build_from_record(@schema, data)
+    end
+
+    private
+
+    def build_from_record(schema, data)
+      schema['fields'].each_with_object({}) do |field, record|
+        field_name = field['name']
+        record[field_name] =
+          case field['type']
+          when 'array'
+            build_from_array(field, data[field_name])
+          when 'map'
+            build_from_map(field, data[field_name])
+          when 'record'
+            build_from_record(field, data[field_name])
+          else
+            build_field(field, data[field_name])
+          end
+      end
+    end
+
+    def build_from_array(field, data)
+      data.map { |element| build_field(field, element) }
+    end
+
+    def build_from_map(field, data)
+      data.transform_values { |element| build_field(field, element) }
+    end
+
+    def build_field(field, datum)
+      check_nullity(datum, field)
+      type = field['items'] || field['values'] || field['type']
+      convert_type(datum, type)
+    end
+
+    def check_nullity(datum, field)
+      type = field['values'] || field['items'] || field['type']
+      unless datum || type&.include?('null')
+        raise ArgumentError.new, "Input for #{field['name']} cannot be nil"
+      end
+    end
+
+    def convert_type(datum, type)
+      type = gather_types(type)
+      if datum.nil?
+        nil
+      elsif type.include?('int') || type.include?('long')
+        datum.to_i
+      elsif type.include?('float') || type.include?('double')
+        datum.to_f
+      elsif type.include?('boolean')
+        datum.casecmp('true').zero?
+      else
+        datum
+      end
+    end
+
+    def gather_types(type)
+      case type
+      when String
+        type
+      when Array
+        type.flat_map { |t| gather_types(t) }
+      when Hash
+        gather_types(type['type'])
+      end
+    end
+  end
+end

data/lib/kril/schema_extractor.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Kril
+  # Extracts avro schemas from avro genrated java files.
+  class SchemaExtractor
+    def initialize(source_dir:, output_dir:)
+      @source_dir = File.join(Dir.pwd, source_dir)
+      @output_dir = File.join(Dir.pwd, output_dir)
+    end
+
+    def extract
+      find_java_files(@source_dir) do |file|
+        schema = parse_avro_java_class(file)
+        write_avsc(schema, @output_dir) if schema
+      end
+    end
+
+    private
+
+    def find_java_files(root_dir)
+      old_dir = Dir.pwd
+      Dir.chdir(root_dir)
+      java_files = File.join('**', '*.java')
+      Dir.glob(java_files) do |file|
+        yield File.new(file)
+      end
+      Dir.chdir(old_dir)
+    end
+
+    def write_avsc(contents, directory)
+      path = File.join(directory, "#{contents['name']}.avsc")
+      File.open(path, 'w') do |file|
+        file.write(JSON.pretty_generate(contents))
+      end
+    end
+
+    def dejavafy(java_string)
+      java_string.split('","').join.gsub(/\\?\\"/, '"')
+    end
+
+    def parse_avro_java_class(file)
+      file.each_line do |line|
+        extraction = line[/SCHEMA.*parse\("(.*)"\);/, 1]
+        break JSON.parse(dejavafy(extraction)) if extraction
+        nil
+      end
+    end
+  end
+end

data/lib/kril/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Kril
+  VERSION = '0.1.0'
+end

data/lib/kril.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'kril/version'
+require 'kril/record_builder'
+require 'kril/schema_extractor'
+require 'kril/producer'
+require 'kril/consumer'
+require 'json'
+require 'yaml'
+require 'logger'
+require 'avro_turf/messaging'
+require 'kafka'
+require 'securerandom'
+
+# TODO
+module Kril
+end