kril 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 67262e75da09ed28a54946538cbf0eede5714481
-  data.tar.gz: d4b669b387aaf0595a28b13e94e1849b9e4ac73b
+  metadata.gz: 273fc08c6713ec195cd1ccb84a067a58f0feeda9
+  data.tar.gz: be5b4f46adbc6202f85a6eb2a99f613654813998
 SHA512:
-  metadata.gz: b2cee45454b6442253c638ad1343eb08660ba82aebcaee08eec0ff971f23d4f53612b6dd0cd6a22f69958f61939cb368565f8525e4404abcda594ba25898af7a
-  data.tar.gz: 894634f416909563340e97ff693401da9a3e2cfff9333a0c1c5166af745c47ce0815bde49b9db13277070dd549345f28dbe3ed327f1e61514b578b3e93904405
+  metadata.gz: 6149bb5ccb97c5c2052af671267703d90a91bc2c1604157213be09dea30035dd7ba3a29926b04c12fc6183d8b2527dbbc8afc5c6beb267291bda2e854a676ddf
+  data.tar.gz: 2ca0265144bcfda279602117da61876b9ed7d52061e316a89f49d8ab1d0906ead3f107849f173bd57f4750b32f5f780030393745ec5c406e8d3fa70644a30ec8

data/.gitignore

@@ -7,4 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
-Gemfile.lock
+schemas/*

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -1,5 +1,5 @@
 source 'https://rubygems.org'
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
 gemspec

data/Gemfile.lock

@@ -0,0 +1,79 @@
+PATH
+  remote: .
+  specs:
+    kril (0.1.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    avro (1.8.2)
+      multi_json
+    avro_turf (0.8.0)
+      avro (>= 1.7.7, < 1.9)
+      excon (~> 0.45)
+    clamp (1.2.1)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.3)
+    excon (0.60.0)
+    hashdiff (0.3.7)
+    httparty (0.16.0)
+      multi_xml (>= 0.5.2)
+    multi_json (1.13.1)
+    multi_xml (0.6.0)
+    mustermann (1.0.2)
+    public_suffix (3.0.2)
+    rack (2.0.4)
+    rack-protection (2.0.1)
+      rack
+    rake (10.5.0)
+    rspec (3.7.0)
+      rspec-core (~> 3.7.0)
+      rspec-expectations (~> 3.7.0)
+      rspec-mocks (~> 3.7.0)
+    rspec-core (3.7.1)
+      rspec-support (~> 3.7.0)
+    rspec-expectations (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-mocks (3.7.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.7.0)
+    rspec-nc (0.3.0)
+      rspec (>= 3)
+      terminal-notifier (>= 1.4)
+    rspec-support (3.7.1)
+    ruby-kafka (0.5.3)
+    safe_yaml (1.0.4)
+    sinatra (2.0.1)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.1)
+      tilt (~> 2.0)
+    terminal-notifier (2.0.0)
+    tilt (2.0.8)
+    webmock (3.3.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  avro_turf (~> 0.8.0)
+  bundler (~> 1.16)
+  clamp (~> 1.2, >= 1.2.1)
+  httparty (~> 0.16.0)
+  kril!
+  rake (~> 10.0)
+  rspec (~> 3.7)
+  rspec-nc (~> 0.3.0)
+  ruby-kafka (~> 0.5.3)
+  sinatra (~> 2.0, >= 2.0.1)
+  webmock (~> 3.3)
+
+BUNDLED WITH
+   1.16.0

data/README.md

@@ -26,12 +26,11 @@ Assuming your schema is not registered with the schema registry, and doesn't exi
 ```bash
 $ kril --bootstrap-servers 'localhost:9092,localhost:9093,localhost:9094' \
 --schema-registry 'http://localhost:8081' \
---with-schema '{"type":"record","name":"human","fields":[{"name":"age","type":"int"}]}' \
+--schema '{"type":"record","name":"human","fields":[{"name":"age","type":"int"}]}' \
 --record '{"age": 27}' \
 human
 ```
 ```bash
-🦐 saved human: {"type"=>"record", "name"=>"human", "fields"=>[{"name"=>"age", "type"=>"int"}]}
 🦐 human: {"age"=>27}
 ```
 
@@ -53,7 +52,7 @@ $ kril --pretty-print human
 }
 ```
 ---
-Now that the schema exists, we can produce records simply:
+Since the schema exists in our repository, we can produce records simply:
 ```bash
 $ kril -r '{"age": 33}' human
 ```
@@ -73,16 +72,31 @@ $ kril --consume-all human
 🦐 human: {:key=>nil, :value=>{"age"=>33}, :offset=>1, :create_time=>2018-03-04 00:34:07 -0700, :topic=>"human", :partition=>3}
 🦐 human: {:key=>nil, :value=>{"age"=>27}, :offset=>0, :create_time=>2018-03-04 00:13:13 -0700, :topic=>"human", :partition=>0}
 ```
+---
+The `--schema` flag is flexible:
+```bash
+$ kril --schema /path/to/schema.avsc
+$ kril --schema name_of_existing_schema
+$ kril --schema '{"type":"record","name":"human","fields":[{"name":"age","type":"int"}]}'
+```
+---
+If no topic is given, the topic will be inferred from the schema name:
+```bash
+$ kril -s human -r '{"age":99}'
+```
+```bash
+🦐 human: {"age"=>99}
+```
 
 ## Contributing
 
 1. Fork it ( https://github.com/ChadBowman/kril/fork )
-2. Create your feature branch (git checkout -b my-new-feature)
-3. Commit your changes (git commit -am 'add some feature')
-4. Push to the branch (git push origin my-new-feature)
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
 5. Create a new Pull Request
 
-Please try to obey [Rubocop](https://github.com/bbatsov/rubocop) to the best of your abilities.
+Please try to obey 👮[Rubocop](https://github.com/bbatsov/rubocop) to the best of your abilities.
 
 ## License
 

data/bin/kril

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
-$VERBOSE = ARGV.map { |arg| true if arg.match?(/-v|--verbose/) }.compact.first
+$VERBOSE = nil
 
 require 'avro_turf/messaging'
 require 'kafka'
@@ -8,72 +8,108 @@ require 'clamp'
 require 'kril'
 
 Clamp do
-  option %w[-k --bootstrap-servers], '', 'address(es) of kafka cluster',
+  $log = Logger.new($STDOUT)
+  $log.level = Logger::ERROR
+
+  option %w[--version], :flag, 'show version' do
+    puts Kril::VERSION
+    exit(0)
+  end
+
+  # configuration
+  option %w[-k --bootstrap-servers], '', 'address(es) of Kafka cluster',
          default: %w[localhost:9092 localhost:9093 localhost:9094] do |address_string|
     address_string.split(/,\s*/)
   end
-  # configuration
   option %w[-g --schema-registry], '', 'address of schema registry', default: 'http://localhost:8081'
-  option %w[-p --schemas-path], '', 'directory of avro schemas', default: 'schemas/'
-  option %w[-v --verbose], :flag, 'print logs, warnings'
-  option %w[-e --pretty-print], :flag, 'pretty print records', default: false
+  option %w[-p --schemas-path], '', 'directory of Avro schemas', default: 'schemas/' do |path|
+    raise ArgumentError, "Schema path: #{path} is not a directory" unless File.directory?(path)
+  end
+  option %w[-v --verbose], :flag, 'print logs, warnings' do
+    $log.level = Logger::DEBUG
+    $VERBOSE = true
+  end
+  option %w[-e --pretty-print], :flag, 'pretty print records'
 
   # producing
   option %w[-r --record], '', 'record to commit to topic'
-  option %w[-o --syncronous], :flag, 'commit records syncronously', default: false
-  option %w[-w --with-schema], '', 'add schema to respository'
+  option %w[-o --synchronous], :flag, 'commit records synchronously'
+  option %w[-s --schema], '', 'schema name, path to schema, or schema contents'
+  option %w[-j --extract-from-java-files], '', 'extract schemas from Avro generated java files'
 
   # consuming
-  option %w[-f --follow], :flag, 'consume from topic indefinitely', defualt: false
-  option %w[-a --consume-all], :flag, 'consume every record on topic', default: false
+  option %w[-a --consume-all], :flag, 'consume every record on topic'
 
-  parameter 'TOPIC', 'topic to produce to or consume from'
-  parameter '[NAME]', 'schema name'
+  parameter '[TOPIC]', 'topic to produce to or consume from'
 
   def execute
-    if topic
-      log = Logger.new($STDOUT)
-      log.level = verbose? ? Logger::WARN : Logger::ERROR
-      kafka = Kafka.new(bootstrap_servers, logger: log, client_id: 'kril')
-      avro = AvroTurf::Messaging.new(registry_url: schema_registry,
-                                     schemas_path: schemas_path,
-                                     logger: log)
-
-      if with_schema
-        schema = JSON.parse(with_schema)
-        schema_name = schema['name']
-        path = File.join(schemas_path, "#{schema_name}.avsc")
-        File.open(path, 'w') { |file| file.write(JSON.pretty_generate(schema)) }
-        print_record("saved #{schema_name}", schema)
+    kafka = Kafka.new(bootstrap_servers, logger: $log, client_id: 'kril')
+    registry = AvroTurf::CachedConfluentSchemaRegistry.new(
+      AvroTurf::ConfluentSchemaRegistry.new(schema_registry, logger: $log)
+    )
+    store = AvroTurf::SchemaStore.new(path: schemas_path)
+    avro = AvroTurf::Messaging.new(registry: registry,
+                                   schema_store: store,
+                                   logger: $log)
+    if extract_from_java_files
+      Kril::SchemaExtractor.extract(source_dir: extract_from_java_files,
+                                    output_dir: schemas_path)
+    end
+    begin
+      if record && (schema || topic)
+        begin
+          produce_record(kafka, avro, store)
+        rescue AvroTurf::SchemaNotFoundError => e
+          print_error(e.message)
+        end
+      elsif topic
+        consume_records(kafka, avro) do |record|
+          print_record(topic, record)
+        end
       end
+    rescue Excon::Error::Socket
+      print_error('could not connect to schema registry')
+    rescue Kafka::ConnectionError => e
+      print_error(e.message)
+    end
+  end
 
-      if record
-        producer = Kril::Producer.new(kafka: kafka, avro: avro)
-        rec = JSON.parse(record)
-        producer.send(record: rec,
-                      schema_name: name || topic,
-                      topic: topic,
-                      syncronous: syncronous?)
-        print_record(topic, rec)
-      else
-        consumer = Kril::Consumer.new(kafka: kafka, avro: avro)
-        if consume_all?
-          consumer.consume_all(topic) do |message|
-            print_record(topic, message) 
-          end
-        elsif follow?
-          consumer.listen(topic) do |message|
-            print_record(topic, message)
-          end
-        else
-          print_record(topic, consumer.consume_one(topic))
-        end
+  private
+
+  def produce_record(kafka, avro, store)
+    producer = Kril::Producer.new(kafka: kafka, avro: avro)
+    schema_name = schema_name(store)
+    record_as_json = JSON.parse(record)
+    infered_topic = topic || schema_name
+    producer.send(record: record_as_json,
+                  schema_name: schema_name,
+                  topic: infered_topic,
+                  syncronous: synchronous?)
+    print_record(infered_topic, record_as_json)
+  end
+
+  def consume_records(kafka, avro)
+    consumer = Kril::Consumer.new(kafka: kafka, avro: avro)
+    if consume_all?
+      consumer.consume_all(topic) do |message|
+        yield message
       end
+    else
+      yield consumer.consume_one(topic)
     end
   end
 
+  def schema_name(schema_store)
+    handler = Kril::SchemaHandler.new(schema_store: schema_store)
+    handler.process(schema || topic)
+  end
+
   def print_record(topic, record)
     record = "\n#{JSON.pretty_generate(record)}" if pretty_print?
     puts "🦐 #{topic}: #{record}"
   end
+
+  def print_error(message)
+    puts "💥 #{message}"
+  end
 end

data/lib/kril/consumer.rb

@@ -1,15 +1,23 @@
 # frozen_string_literal: true
 
 module Kril
-  # Consumers records from Kafka
+  # High level abstraction for consuming records from topics.
   class Consumer
+    # avro   - Avro instance for deserializing records [AvroTurf::Messaging]
+    # kafka  - Kafka instance for creating consumers [Kafka]
+    # config - consumer configuration (optional) [Hash]
     def initialize(avro: nil, kafka: nil, config: {})
-      config[:group_id] ||= '🦐'
+      config[:group_id] ||= 'kril-consumer'
       @avro = avro
       @kafka = kafka
       @config = config
     end
 
+    # Consume a single record from any partition.
+    # Will block indefinitely if no record present.
+    #
+    # topic  - topic to consume from [String]
+    # return - deserialized record [String]
     def consume_one(topic)
       consumer = build_consumer(topic, true, @config)
       msg = nil
@@ -24,6 +32,13 @@ module Kril
       consumer.stop
     end
 
+    # Consume all records from a topic. Each record will be yielded
+    # to block along with consumer instance. Will listen to topic
+    # after all records have been consumed.
+    #
+    # topic  - topic to consume from [String]
+    # yields - record, consumer [String, Kafka::Consumer]
+    # return - [nil]
     def consume_all(topic)
       config = @config.clone
       config[:group_id] = SecureRandom.uuid
@@ -35,15 +50,6 @@ module Kril
       consumer.stop
     end
 
-    def listen(topic)
-      consumer = build_consumer(topic, false, @config)
-      consumer.each_message do |message|
-        yield decode(message), consumer
-      end
-    ensure
-      consumer.stop
-    end
-
     private
 
     def build_consumer(topic, start_from_beginning, config)

data/lib/kril/producer.rb

@@ -1,8 +1,11 @@
 # frozen_string_literal: true
 
 module Kril
-  # Produces records to Kafka
+  # High level abstraction for producing records to topics.
   class Producer
+    # avro   - Avro instance for deserializing records [AvroTurf::Messaging]
+    # kafka  - Kafka instance for creating producers [Kafka]
+    # config - producer configuration (optional) [Hash]
     def initialize(avro: nil, kafka: nil, config: {})
       config[:required_acks] ||= 1
       config[:delivery_threshold] ||= 1
@@ -13,12 +16,18 @@ module Kril
       @sync = kafka.producer(sync_config)
     end
 
+    # Commit a record to a topic.
+    #
+    # record       - record to serialize and commit [String]
+    # schema_name  - name of schema to encode record from [String]
+    # topic        - name of topic. Will be schema_name if nil (optional) [String]
+    # synchronous  - blocks until commit if true (optional) [Boolean]
     def send(record:, schema_name:, topic: nil, syncronous: false)
       topic ||= schema_name
       encoded = @avro.encode(record, schema_name: schema_name)
       if syncronous
-        @producer.produce(encoded, topic: topic)
-        @producer.deliver_messages
+        @sync.produce(encoded, topic: topic)
+        @sync.deliver_messages
       else
         @async.produce(encoded, topic: topic)
       end

data/lib/kril/record_builder.rb

@@ -4,7 +4,7 @@ module Kril
   # Creates and validates a record based from a defined schema.
   class RecordBuilder
     def initialize(schema_file_name: nil,
-                   schemas_dir: 'schemas',
+                   schemas_dir: 'schemas/',
                    schema_path: nil)
       path = schema_path || File.join(schemas_dir, "#{schema_file_name}.avsc")
       file = File.read(path)

data/lib/kril/schema_extractor.rb

@@ -1,21 +1,24 @@
 # frozen_string_literal: true
 
 module Kril
-  # Extracts avro schemas from avro genrated java files.
-  class SchemaExtractor
-    def initialize(source_dir:, output_dir:)
-      @source_dir = File.join(Dir.pwd, source_dir)
-      @output_dir = File.join(Dir.pwd, output_dir)
-    end
-
-    def extract
-      find_java_files(@source_dir) do |file|
+  # Extracts schemas from avro generated java files.
+  module SchemaExtractor
+    # Parse schemas from avro generated java files and
+    # load them into the schema repository.
+    #
+    # source_dir - root directory of java files [String]
+    # output_dir - schema repository [String]
+    # returns    - [nil]
+    def self.extract(source_dir:, output_dir:)
+      output_dir = File.join(Dir.pwd, output_dir)
+      find_java_files(source_dir) do |file|
         schema = parse_avro_java_class(file)
-        write_avsc(schema, @output_dir) if schema
+        write_avsc(schema, output_dir) if schema
       end
+      nil
     end
 
-    private
+    module_function
 
     def find_java_files(root_dir)
       old_dir = Dir.pwd
@@ -39,11 +42,11 @@ module Kril
     end
 
     def parse_avro_java_class(file)
-      file.each_line do |line|
+      result = file.each_line do |line|
         extraction = line[/SCHEMA.*parse\("(.*)"\);/, 1]
         break JSON.parse(dejavafy(extraction)) if extraction
-        nil
       end
+      result.is_a?(File) ? nil : result
     end
   end
 end

data/lib/kril/schema_handler.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Kril
+  # Saves schemas to repository.
+  class SchemaHandler
+    # schemas_path - directory of schema repository [String]
+    # schema_store - schema store [AvroTurf::SchemaStore]
+    def initialize(schemas_path: 'schemas/',
+                   schema_store: nil)
+      schema_store ||= AvroTurf::SchemaStore.new(path: schemas_path)
+      @schema_store = schema_store
+      @schemas_path = schemas_path
+    end
+
+    # Handles input to reference or create schema.
+    #
+    # input_string - schema name, schema file, or schema contents [String]
+    # returns      - schema name [String]
+    def process(input_string)
+      if File.exist?(input_string)
+        copy_schema_to_store(input_string)
+      elsif schema?(input_string)
+        save_schema(input_string)
+      else
+        @schema_store.find(input_string).name
+      end
+    end
+
+    private
+
+    def schema?(input)
+      JSON.parse(input)['name']
+    rescue StandardError
+      false
+    end
+
+    def save_schema(schema)
+      schema = JSON.parse(schema)
+      schema_name = schema['name']
+      path = File.join(@schemas_path, "#{schema_name}.avsc")
+      File.open(path, 'w') { |file| file.write(JSON.pretty_generate(schema)) }
+      schema_name
+    end
+
+    def copy_schema_to_store(path)
+      schema = File.read(path)
+      raise ArgumentError.new, "Not a valid schema: #{path}" unless schema?(schema)
+      schema_name = JSON.parse(schema)['name']
+      new_path = File.join(@schemas_path, "#{schema_name}.avsc")
+      FileUtils.copy_file(path, new_path)
+      schema_name
+    end
+  end
+end

data/lib/kril/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kril
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

data/lib/kril.rb

@@ -3,6 +3,7 @@
 require 'kril/version'
 require 'kril/record_builder'
 require 'kril/schema_extractor'
+require 'kril/schema_handler'
 require 'kril/producer'
 require 'kril/consumer'
 require 'json'
@@ -11,7 +12,11 @@ require 'logger'
 require 'avro_turf/messaging'
 require 'kafka'
 require 'securerandom'
+require 'fileutils'
 
-# TODO
+# Simple, easy to use API for interacting with
+# Apache Kafka with Apache Avro. Intended for
+# experimentation in development, as nothing here
+# is optimized for performance.
 module Kril
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kril
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Chad Bowman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-04 00:00:00.000000000 Z
+date: 2018-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf
@@ -175,8 +175,10 @@ files:
 - ".idea/modules.xml"
 - ".idea/vcs.xml"
 - ".idea/workspace.xml"
+- ".rspec"
 - ".travis.yml"
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -187,6 +189,7 @@ files:
 - lib/kril/producer.rb
 - lib/kril/record_builder.rb
 - lib/kril/schema_extractor.rb
+- lib/kril/schema_handler.rb
 - lib/kril/version.rb
 homepage: https://github.com/ChadBowman/kril
 licenses: