kafka_rest_client 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f49da809a9a1da43ec33c8d475ff31055ca46cd
-  data.tar.gz: f75cc6a67c1806727403dc59f7bb755566fa680c
+  metadata.gz: a2547fd6407a534cd9dcc57623b629f2a4dc3f67
+  data.tar.gz: 46678ca71d81e2ab8d4618c54ef6ae89ac1a912c
 SHA512:
-  metadata.gz: 9cc1954c043763172bc80baed3da9457e504c8e9e231a59d8711d6938e83c792b26be9f4fdc1d222522a0c9a781737bff605120b97420faec9efe05a32270729
-  data.tar.gz: 186bed143363206e04128674fa49b91d4c0ca621261599a3fa9f82816de7293fa00f4c1a726b9a6b8f5345ff1fad0261bbd60e95e1d64babc9e8b4f762b780f1
+  metadata.gz: ebf98c38adac856ae66952a02081451aebcac5f3406ce157c7656719dd4582ebce5ef39eecc65623b90349c5b92a83f25f92054d14ae32c7c0187c1eb1b87649
+  data.tar.gz: 69768766374ee6d1d765251888e5c9dae0173470fdccaea8015681768b33743ffd08f3814459ef75d1dc30a9fd5ec22402113a569421b1588470fbb0e30cc309

data/README.md

@@ -2,7 +2,7 @@
 
 A Ruby client to interact with [Kafka REST Proxy](http://docs.confluent.io/1.0.1/kafka-rest/docs/index.html)
 
-**Current Version:** 0.2.1
+**Current Version:** 0.3.0
 
 **Supported Ruby versions:** 2.0, 2.1, 2.2
 
@@ -11,7 +11,7 @@ A Ruby client to interact with [Kafka REST Proxy](http://docs.confluent.io/1.0.1
 Add this line to your application's Gemfile:
 
 ```ruby
-gem 'kafka_rest_client', git: 'git@github.com:FundingCircle/kafka_rest_client.git', tag: 'v0.2.1'
+gem 'kafka_rest_client', git: 'git@github.com:FundingCircle/kafka_rest_client.git', tag: 'v0.3.0'
 ```
 
 ## Usage
@@ -39,6 +39,9 @@ producer = KafkaRestClient::AvroProducer.new
 # with the "#{topic}-value" name
 producer.produce('ice-cream-melted', { temperature: 35, unit: 'celsius' })
 
+# This would post a request to the REST Proxy e.g. :
+# {"id": 1, "temperature": 32, "unit": "celsius"}
+
 # Produce multiple events
 # The schema_id will be fetched from the schema registry by looking up a schema
 # with the "#{topic}-value" name
@@ -70,6 +73,26 @@ end
 
 producer.produce('ice-cream-melted', [IceCreamMeltedEvent.new(35, 'celsius')])
 
+# All of the above examples expect events to have union types explicitly defined. 
+# For example, if you have a nullable string field, the type annotation needs to be: 
+# {"field_name": { "string": "field_value"}}
+# You can provide the following option to automatically annotate nullable fields.
+# ⚠️ This does not currently support nested documents
+producer.produce('ice-cream-melted',
+                 { temperature: 35, unit: 'celsius' },
+                 annotate_optional_fields: true,
+                 value_schema: {"namespace": "com.fundingcircle",
+                                "type": "record",
+                                "name": "IceCreamMelted",
+                                "fields": [{"name": "id", "type": "string"},
+                                           {"name": "temperature","type": "int"},
+                                           {"name": "unit",
+                                            "type": ["null", "string"]
+                                           }]
+                 })
+# This would post a request to the REST Proxy with the correct type annotation, e.g. :
+# {"id": 1, "temperature": 32, "unit": { "string": "celsius"}}
+
 # Exception handling
 begin
   producer.produce('ice-cream-melted', { temperature: 35, unit: 'celsius' })

data/kafka_rest_client.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'kafka_rest_client'
-  spec.version       = '0.2.2'
+  spec.version       = '0.3.0'
   spec.authors       = ['Funding Circle Engineering']
   spec.email         = ['engineering+kafka_rest_client@fundingcircle.com']
   spec.licenses      = ['BSD-3-Clause']

data/lib/kafka_rest_client/avro_producer.rb

@@ -1,7 +1,7 @@
-require 'kafka_rest_client/errors'
 require 'kafka_rest_client/configuration'
+require 'kafka_rest_client/errors'
+require 'kafka_rest_client/utils'
 require 'httparty'
-require 'json'
 
 module KafkaRestClient
   class AvroProducer
@@ -24,7 +24,11 @@ module KafkaRestClient
 
     def produce(topic, events, options = {})
       if enabled?
-        payload  = build_event_payload(topic, [events].flatten, options)
+        events = [events].flatten.map { |event| event_to_hash event }
+        if options[:annotate_optional_fields]
+          serialized_events = annotate_events_optional_fields(topic, events, options)
+        end
+        payload  = build_event_payload(topic, (serialized_events || events), options)
         response = post_event_to_kafka(topic, payload)
         config.logger.debug("Produced to Kafka topic #{topic}: #{payload.inspect}")
         response
@@ -33,6 +37,52 @@ module KafkaRestClient
 
     private
 
+    def event_to_hash(event)
+      if event.respond_to? :as_json
+        event.as_json
+      else
+        Utils.deep_stringify_keys event
+      end
+    end
+
+    def annotate_optional_fields(event, fields)
+      event.each do |k, v|
+        if v.is_a?(Hash)
+          fail NestedHashSerializationNotImplemented
+        elsif v && fields[k]
+          event[k] = { fields[k]['type'] => event[k] }
+        end
+      end
+    end
+
+    def select_schema(topic, options)
+      if options[:value_schema]
+        options[:value_schema]
+      elsif options[:value_schema_id]
+        get_schema_by_id(options[:value_schema_id]).fetch('schema')
+      else
+        get_latest_schema(topic).fetch('schema')
+      end
+    end
+
+    # Transforms an optional schema attribute
+    # From {"name"=>"temperature", "type"=>["null", "double"]}
+    # To {"temperature"=>{"type"=>"double"}}
+    def schema_optional_fields(raw_schema)
+      JSON.parse(raw_schema).fetch('fields').inject({}) do |hash, attribute|
+        if attribute['type'].include? 'null'
+          hash[attribute['name']] = { 'type' => (attribute['type'] - ['null']).first }
+        end
+        hash
+      end
+    end
+
+    def annotate_events_optional_fields(topic, events, options)
+      raw_schema = select_schema(topic, options)
+      optional_fields = schema_optional_fields(raw_schema)
+      events.map { |event| annotate_optional_fields(event, optional_fields) }
+    end
+
     def build_event_payload(topic, events, options)
       payload = { records: events.map { |event| { value: event } } }.merge(
         options.select { |k, _| [:value_schema_id, :value_schema].include?(k) }
@@ -45,16 +95,13 @@ module KafkaRestClient
       payload
     end
 
-    def get_latest_schema(topic)
-      response = HTTParty.get(
-        "#{config.schema_registry_url}/subjects/#{topic}-value/versions/latest",
-        timeout: config.timeout
-      )
+    def get_schema(id, url)
+      response = HTTParty.get(url, timeout: config.timeout)
 
       if response.code == 200
         JSON.parse(response.body)
       elsif response.code == 404
-        fail SchemaNotFoundError, "Schema for #{topic} not found: #{response.body}"
+        fail SchemaNotFoundError, "Schema for #{id} not found: #{response.body}"
       elsif response.code == 500
         fail InternalServerError, "Schema registry internal error: #{response.body}"
       elsif response.code == 503
@@ -64,6 +111,16 @@ module KafkaRestClient
       end
     end
 
+    def get_schema_by_id(id)
+      url = "#{config.schema_registry_url}/schemas/ids/#{id}"
+      get_schema(id, url)
+    end
+
+    def get_latest_schema(topic)
+      url = "#{config.schema_registry_url}/subjects/#{topic}-value/versions/latest"
+      get_schema('latest', url)
+    end
+
     def post_event_to_kafka(topic, message)
       response = HTTParty.post(
         "#{config.kafka_proxy_url}/topics/#{topic}",

data/lib/kafka_rest_client/errors.rb

@@ -16,4 +16,6 @@ module KafkaRestClient
 
   # Exception raised on a bad gateway proxy resolution failure (503)
   class BadGateway < Error; end
+
+  class NestedHashSerializationNotImplemented < Error; end
 end

data/lib/kafka_rest_client/utils.rb

@@ -0,0 +1,26 @@
+module KafkaRestClient
+  module Utils
+    # Courtersy of avdi
+    # Source https://gist.github.com/avdi/239567
+    def transform_hash(original, options={}, &block)
+      original.inject({}){|result, (key,value)|
+        value = if (options[:deep] && Hash === value)
+                  transform_hash(value, options, &block)
+                else
+                  value
+                end
+        block.call(result,key,value)
+        result
+      }
+    end
+
+    def deep_stringify_keys(hash)
+      transform_hash(hash, :deep => true) {|hash, key, value|
+        hash[key.to_s] = value
+      }
+    end
+
+    module_function :transform_hash
+    module_function :deep_stringify_keys
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kafka_rest_client
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Funding Circle Engineering
@@ -144,6 +144,7 @@ files:
 - lib/kafka_rest_client/avro_producer.rb
 - lib/kafka_rest_client/configuration.rb
 - lib/kafka_rest_client/errors.rb
+- lib/kafka_rest_client/utils.rb
 homepage: http://github.com/FundingCircle/kakfa_rest_client
 licenses:
 - BSD-3-Clause