kril 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f29ebc51e511a7e37aee1e470e17656983b6a76a
-  data.tar.gz: 694569edd72f95f312416fc130980c568eea2ed3
+  metadata.gz: dc5623bc2f2039f53fbfe78a9fdf24b9aa14ec73
+  data.tar.gz: 05fad289a66fa8886113ec829d1336ba956bc1e5
 SHA512:
-  metadata.gz: 5f5ce6ac018415bbccca2ef13c158d1359b670133d1207bc77d0c961f19df1ae56d2a0016c01134e92263443f715aea0ab8b2cc7aba067269d287d8540838498
-  data.tar.gz: 6f5848ff5d4bafb165bfed72ba235627f45ef45ab2a35d17a18907f30de7a476b15b39c7fd1230e0f68d90e194e53f00db42426e1a5dea115e4338db09537b29
+  metadata.gz: 43ad965f8c1c59a0210f80216e127bccc68e631fca4dd2feaf438c3550817ce3d4b377bf3941c50560e3ef830c1f158378e62e4ee7283f0d9b2b16f6106287de
+  data.tar.gz: b4025a0d14a3e16fa3000908627b8120244f9664064477f366d0779f7c9318bc5a03a751a336985caca32855296dfe85effd54c77337134fdade1caf076e5d3a

data/.gitignore

@@ -7,4 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
-*.avsc
+/schemas

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kril (0.1.4)
+    kril (0.1.5)
       avro_turf (~> 0.8.0)
       clamp (~> 1.2, >= 1.2.1)
       ruby-kafka (~> 0.5.3)

data/README.md

@@ -73,7 +73,7 @@ $ kril --consume-all human
 🦐 human: {:key=>nil, :value=>{"age"=>27}, :offset=>0, :create_time=>2018-03-04 00:13:13 -0700, :topic=>"human", :partition=>0}
 ```
 ---
-The `--schema` flag is flexible:
+The `--schema` option is flexible:
 ```bash
 $ kril --schema /path/to/schema.avsc
 $ kril --schema name_of_existing_schema
@@ -87,7 +87,14 @@ $ kril -s human -r '{"age":99}'
 ```bash
 🦐 human: {"age"=>99}
 ```
-
+---
+To see what schemas are saved in the repository:
+```bash
+$ kril --list-schemas
+```
+```bash
+human, another_schema
+```
 ## Contributing
 
 1. Fork it ( https://github.com/ChadBowman/kril/fork )
@@ -96,7 +103,7 @@ $ kril -s human -r '{"age":99}'
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create a new Pull Request
 
-Please try to obey 👮[Rubocop](https://github.com/bbatsov/rubocop) to the best of your abilities.
+Obey 👮[Rubocop](https://github.com/bbatsov/rubocop)! 🚨
 
 ## License
 

data/bin/kril

@@ -10,7 +10,7 @@ require 'kril'
 Clamp do
   $log = Logger.new($STDOUT)
   $log.level = Logger::ERROR
-  gem_root = File.expand_path('..', __dir__)
+  $gem_root = File.expand_path('..', __dir__)
 
   option %w[--version], :flag, 'show version' do
     puts Kril::VERSION
@@ -23,7 +23,7 @@ Clamp do
     address_string.split(/,\s*/)
   end
   option %w[-g --schema-registry], '', 'address of schema registry', default: 'http://localhost:8081'
-  option %w[-p --schemas-path], '', 'directory of Avro schemas', default: "#{File.join(gem_root, 'schemas')}" do |path|
+  option %w[-p --schemas-path], '', 'directory of Avro schemas', default: File.join($gem_root, 'schemas').to_s do |path|
     raise ArgumentError, "Schema path: #{path} is not a directory" unless File.directory?(path)
   end
   option %w[-v --verbose], :flag, 'print logs, warnings' do
@@ -36,11 +36,14 @@ Clamp do
   option %w[-r --record], '', 'record to commit to topic'
   option %w[-o --synchronous], :flag, 'commit records synchronously'
   option %w[-s --schema], '', 'schema name, path to schema, or schema contents'
-  option %w[-j --extract-from-java-files], '', 'extract schemas from Avro generated java files'
+  option %w[-j --extract-schemas], '', 'extract schemas from Avro generated java files'
 
   # consuming
   option %w[-a --consume-all], :flag, 'consume every record on topic'
 
+  # utility
+  option %w[-l --list-schemas], :flag, 'list saved schemas'
+
   parameter '[TOPIC]', 'topic to produce to or consume from'
 
   def execute
@@ -52,7 +55,7 @@ Clamp do
     avro = AvroTurf::Messaging.new(registry: registry,
                                    schema_store: store,
                                    logger: $log)
-    if extract_from_java_files
+    if extract_schemas
       Kril::SchemaExtractor.extract(source_dir: extract_from_java_files,
                                     output_dir: schemas_path)
     end
@@ -73,17 +76,22 @@ Clamp do
     rescue Kafka::ConnectionError => e
       print_error(e.message)
     end
+    if list_schemas?
+      schemas = File.join($gem_root, 'schemas', '**', '*.avsc')
+      puts Dir.glob(schemas).map { |file| File.basename(file, '.avsc') }.join(', ')
+    end
   end
 
   private
 
   def produce_record(kafka, avro, store)
     producer = Kril::Producer.new(kafka: kafka, avro: avro)
-    schema_name = schema_name(store)
+    names = schema_name(store)
     record_as_json = JSON.parse(record)
-    infered_topic = topic || schema_name
+    infered_topic = topic || names[:schema_name]
     producer.send(record: record_as_json,
-                  schema_name: schema_name,
+                  schema_name: names[:schema_name],
+                  namespace: names[:namespace],
                   topic: infered_topic,
                   syncronous: synchronous?)
     print_record(infered_topic, record_as_json)
@@ -101,7 +109,7 @@ Clamp do
   end
 
   def schema_name(schema_store)
-    handler = Kril::SchemaHandler.new(schema_store: schema_store)
+    handler = Kril::SchemaHandler.new(schema_store: schema_store, schemas_path: schemas_path)
     handler.process(schema || topic)
   end
 

data/lib/kril/producer.rb

@@ -20,11 +20,18 @@ module Kril
     #
     # record       - record to serialize and commit [String]
     # schema_name  - name of schema to encode record from [String]
+    # namespace    - namespace of schema (optional) [String]
     # topic        - name of topic. Will be schema_name if nil (optional) [String]
     # synchronous  - blocks until commit if true (optional) [Boolean]
-    def send(record:, schema_name:, topic: nil, syncronous: false)
+    def send(record:,
+             schema_name:,
+             namespace: nil,
+             topic: nil,
+             syncronous: false)
       topic ||= schema_name
-      encoded = @avro.encode(record, schema_name: schema_name)
+      encoded = @avro.encode(record,
+                             schema_name: schema_name,
+                             namespace: namespace)
       if syncronous
         @sync.produce(encoded, topic: topic)
         @sync.deliver_messages

data/lib/kril/schema_handler.rb

@@ -5,7 +5,7 @@ module Kril
   class SchemaHandler
     # schemas_path - directory of schema repository [String]
     # schema_store - schema store [AvroTurf::SchemaStore]
-    def initialize(schemas_path: nil,
+    def initialize(schemas_path:,
                    schema_store: nil)
       schema_store ||= AvroTurf::SchemaStore.new(path: schemas_path)
       @schema_store = schema_store
@@ -15,21 +15,32 @@ module Kril
     # Handles input to reference or create schema.
     #
     # input_string - schema name, schema file, or schema contents [String]
-    # returns      - schema name [String]
+    # returns      - schema name, namespace [Hash]
     def process(input_string)
       if File.exist?(input_string)
         copy_schema_to_store(input_string)
       elsif schema?(input_string)
         save_schema(input_string)
       else
-        @schema_store.find(input_string).name
+        name, namespace = separate_fullname(input_string)
+        namespace = nil if namespace.empty?
+        schema = @schema_store.find(name, namespace)
+        {
+          schema_name: schema&.name,
+          namespace: schema&.namespace
+        }
       end
     end
 
     private
 
+    def separate_fullname(fullname)
+      arr = fullname.split('.')
+      [arr.pop, arr.join('.')]
+    end
+
     def schema?(input)
-      JSON.parse(input)['name']
+      !JSON.parse(input)['name'].nil?
     rescue StandardError
       false
     end
@@ -37,18 +48,35 @@ module Kril
     def save_schema(schema)
       schema = JSON.parse(schema)
       schema_name = schema['name']
-      path = File.join(@schemas_path, "#{schema_name}.avsc")
-      File.open(path, 'w') { |file| file.write(JSON.pretty_generate(schema)) }
-      schema_name
+      namespace = schema['namespace']
+      if namespace
+        path = File.join(@schemas_path, namespace.split('.'))
+        FileUtils.mkdir_p(path)
+      end
+      File.open(File.join(path || @schemas_path, "#{schema_name}.avsc"), 'w') do |file|
+        file.write(JSON.pretty_generate(schema))
+      end
+      {
+        schema_name: schema_name,
+        namespace: namespace
+      }
     end
 
     def copy_schema_to_store(path)
       schema = File.read(path)
       raise ArgumentError, "Not a valid schema: #{path}" unless schema?(schema)
-      schema_name = JSON.parse(schema)['name']
-      new_path = File.join(@schemas_path, "#{schema_name}.avsc")
-      FileUtils.copy_file(path, new_path)
-      schema_name
+      json = JSON.parse(schema)
+      schema_name = json['name']
+      namespace = json['namespace']
+      if namespace
+        schema_path = File.join(@schemas_path, namespace.split('.'))
+        FileUtils.mkdir_p(schema_path)
+      end
+      FileUtils.copy_file(path, File.join(schema_path || @schemas_path, "#{schema_name}.avsc"))
+      {
+        schema_name: schema_name,
+        namespace: namespace
+      }
     end
   end
 end

data/lib/kril/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kril
-  VERSION = '0.1.5'
+  VERSION = '0.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kril
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Chad Bowman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-05 00:00:00.000000000 Z
+date: 2018-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro_turf