kafka_rest_client 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 55eaec42836887d93b9460701cdca823c0f63f77
+  data.tar.gz: b6eeb11b3fa7c973c2b3d51fe00631f5b92b63d6
+SHA512:
+  metadata.gz: d9af06c6d72c0284905b76883f4dfc338eae7c6d39bfdb54ffefede69cd8b6462b1ac761085fced8340d9a274b7f76a46ae0ded6d59c5979fe13b9a1f49fa456
+  data.tar.gz: 3ac7cd879e202aea4ef4010099f3816455cd796d9545b316768a54098905a57e8b30f915fdb50acaae0b1f02f9b264d9f48b7e3e49261e4c2c0b977927366c3c

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,4 @@
+Style/Documentation:
+  Enabled: false
+Metrics/LineLength:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+2.0.0-p645

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in kafka_rest_client.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,45 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec features) \
+#  .select{|d| Dir.exists?(d) ? d : UI.warning("Directory #{d} does not exist")}
+
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), then you will want to move
+## the Guardfile to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+# Note: The cmd option is now required due to the increasing number of ways
+#       rspec may be run, below are examples of the most common uses.
+#  * bundler: 'bundle exec rspec'
+#  * bundler binstubs: 'bin/rspec'
+#  * spring: 'bin/rspec' (This will use spring if running and you have
+#                          installed the spring binstubs per the docs)
+#  * zeus: 'zeus rspec' (requires the server to be started separately)
+#  * 'just' rspec: 'rspec'
+
+guard :rspec, cmd: 'bundle exec rspec' do
+  require 'guard/rspec/dsl'
+  dsl = Guard::RSpec::Dsl.new(self)
+
+  # RSpec files
+  rspec = dsl.rspec
+  watch(rspec.spec_helper) { rspec.spec_dir }
+  watch(rspec.spec_support) { rspec.spec_dir }
+  watch(rspec.spec_files)
+
+  # Ruby files
+  ruby = dsl.ruby
+  dsl.watch_spec_files_for(ruby.lib_files)
+end
+
+guard :rubocop do
+  watch(/.+\.rb$/)
+  watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
+end

data/README.md

@@ -0,0 +1,68 @@
+# KafkaRestClient
+
+A Ruby client to interact with [Kafka REST Proxy](http://docs.confluent.io/1.0.1/kafka-rest/docs/index.html)
+
+**Current Version:** 0.1.0
+
+**Supported Ruby versions:** 2.0, 2.1, 2.2
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'kafka_rest_client', git: 'git@github.com:FundingCircle/kafka_rest_client.git', tag: 'v0.1.0'
+```
+
+## Usage
+
+### Producing events
+```ruby
+require 'kafka_rest_client'
+
+# Configure global settings
+KafkaRestClient.configure do |config|
+  config.kafka_proxy_url = ENV['KAFKA_PROXY_URL']
+  config.schema_registry_url = ENV['SCHEMA_REGISTRY_URL']
+  config.timeout = 10
+  config.logger = Rails.logger
+end
+
+producer = KafkaRestClient::AvroProducer.new
+
+# Produce a single event using the topic name and payload
+producer.produce('ice-cream-melted', { temperature: 35, unit: 'celsius' })
+
+# Produce multiple events
+producer.produce('ice-cream-melted', [{ temperature: 35, unit: 'celsius' }])
+
+# Produce event objects by relying on #to_json and #as_json when using Rails ActiveSupport
+class IceCreamMeltedEvent
+  attr_reader :temperature, :unit
+  def initialize(temperature, unit)
+    @temperature = temperature
+    @unit = unit
+  end
+
+  def as_json(_options = nil)
+    { temperature: temperature, unit: unit }
+  end
+end
+
+producer.produce('ice-cream-melted', [IceCreamMeltedEvent.new(35, 'celsius')])
+
+# Exception handling
+begin
+  producer.produce('ice-cream-melted', { temperature: 35, unit: 'celsius' })
+rescue KafkaRestClient::SchemaNotFoundError => e
+ # Schema has not been registered in the schema registry
+rescue KafkaRestClient::TopicNotFoundError => e
+ # Topic does not exist in Kafka, create it with confluent tools
+rescue KafkaRestClient::Error => e
+ # Rescue any error raised by the library
+end
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/fundingcircle/kafka_rest_client.

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
+
+task default: [:spec, :rubocop]

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'kafka_rest_client'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require 'pry'
+Pry.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/kafka_rest_client.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'kafka_rest_client'
+  spec.version       = '0.1.0'
+  spec.authors       = ['Funding Circle Engineering']
+  spec.email         = ['engineering+kafka_rest_client@fundingcircle.com']
+
+  spec.summary       = 'Ruby client for interacting with Kafka REST Proxy'
+  spec.description   = 'Ruby client for interacting with Kafka REST Proxy'
+  spec.homepage      = 'http://github.com/FundingCircle/kakfa_rest_client'
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f|
+    f.match(%r{^(test|spec|features)/})
+  }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.10'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.4'
+  spec.add_development_dependency 'pry', '~> 0.10'
+  spec.add_development_dependency 'guard-rspec', '~> 4.6'
+  spec.add_development_dependency 'rubocop', '~> 0.36'
+  spec.add_development_dependency 'guard-rubocop', '~> 1.2'
+  spec.add_development_dependency 'webmock', '~> 1.22'
+  spec.add_dependency 'httparty', '~> 0.13'
+end

data/lib/kafka_rest_client.rb

@@ -0,0 +1,8 @@
+require 'kafka_rest_client/avro_producer'
+
+module KafkaRestClient
+  def self.configure
+    Configuration.current = Configuration.new
+    yield(Configuration.current)
+  end
+end

data/lib/kafka_rest_client/avro_producer.rb

@@ -0,0 +1,89 @@
+require 'kafka_rest_client/errors'
+require 'kafka_rest_client/configuration'
+require 'httparty'
+require 'json'
+
+module KafkaRestClient
+  class AvroProducer
+    AVRO_AS_JSON = 'application/vnd.kafka.avro.v1+json'.freeze
+
+    attr_reader :config
+    private :config
+
+    def initialize(config = Configuration.current)
+      @config = config
+    end
+
+    def enabled?
+      if config.kafka_proxy_url && config.schema_registry_url
+        true
+      else
+        false
+      end
+    end
+
+    def produce(topic, events)
+      if enabled?
+        payload  = build_event_payload(topic, Array(events))
+        response = post_event_to_kafka(topic, payload)
+        config.logger.debug("Produced to Kafka topic #{topic}: #{payload.inspect}")
+        response
+      end
+    end
+
+    private
+
+    def build_event_payload(topic, events)
+      {
+        value_schema_id: get_latest_schema(topic).fetch('id'),
+        records: events.map { |event| { value: event } }
+      }
+    end
+
+    def get_latest_schema(topic)
+      response = HTTParty.get(
+        "#{config.schema_registry_url}/subjects/#{topic}-value/versions/latest",
+        timeout: config.timeout
+      )
+
+      if response.code == 200
+        JSON.parse(response.body)
+      elsif response.code == 404
+        fail SchemaNotFoundError, "Schema for #{topic} not found: #{response.body}"
+      elsif response.code == 500
+        fail InternalServerError, "Schema registry internal error: #{response.body}"
+      elsif response.code == 503
+        fail BadGateway, "Bad gateway: #{response.body}"
+      else
+        fail "Unexpected error: #{response.body}"
+      end
+    end
+
+    def post_event_to_kafka(topic, message)
+      response = HTTParty.post(
+        "#{config.kafka_proxy_url}/topics/#{topic}",
+        headers: { 'Content-Type' => AVRO_AS_JSON },
+        body: message.to_json,
+        timeout: config.timeout
+      )
+
+      if response.code == 200
+        config.logger.debug("#{message[:records].count} event(s) published on #{topic}")
+        JSON.parse(response.body)
+      elsif response.code == 404
+        fail SchemaNotFoundError,
+             "Topic #{topic} not found in schema registry"
+      elsif response.code == 422
+        fail SchemaValidationError,
+             "Message #{message} does not match schema for #{topic} or is otherwise invalid"
+      elsif response.code == 500
+        fail InternalServerError,
+             'Kafka Rest Proxy internal error (check zookeeper, kafka services)'
+      elsif response.code == 503
+        fail BadGateway, "Bad gateway: #{response.body}"
+      else
+        fail "Unexpected error: #{response.body}"
+      end
+    end
+  end
+end

data/lib/kafka_rest_client/configuration.rb

@@ -0,0 +1,28 @@
+require 'logger'
+
+module KafkaRestClient
+  class Configuration
+    attr_accessor :kafka_proxy_url, :schema_registry_url, :timeout, :logger
+    class << self
+      attr_accessor :current
+    end
+
+    def initialize(options = {})
+      @kafka_proxy_url = options[:kafka_proxy_url]
+      @schema_registry_url = options[:schema_registry_url]
+      @timeout = options[:timeout]
+      @logger = options[:loggger]
+    end
+
+    def timeout
+      @timeout ||= 30
+    end
+
+    def logger
+      @logger ||= Logger.new(STDOUT).tap do |logger|
+        # Set level too high so we don't log anything
+        logger.level = Logger::FATAL
+      end
+    end
+  end
+end

data/lib/kafka_rest_client/errors.rb

@@ -0,0 +1,16 @@
+module KafkaRestClient
+  # Base Error for all error classes of the library
+  class Error < StandardError; end
+
+  # Exception raised when the desired schema is not found (404 response code)
+  class SchemaNotFoundError < Error; end
+
+  # Exception raised on an internal server error (500 response code)
+  class InternalServerError < Error; end
+
+  # Exception raised on a message not matching the requested schema (422)
+  class SchemaValidationError < Error; end
+
+  # Exception raised on a bad gateway proxy resolution failure (503)
+  class BadGateway < Error; end
+end

metadata

@@ -0,0 +1,184 @@
+--- !ruby/object:Gem::Specification
+name: kafka_rest_client
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Funding Circle Engineering
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-06-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.6'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.36'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.36'
+- !ruby/object:Gem::Dependency
+  name: guard-rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.22'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+description: Ruby client for interacting with Kafka REST Proxy
+email:
+- engineering+kafka_rest_client@fundingcircle.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- Gemfile
+- Guardfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- kafka_rest_client.gemspec
+- lib/kafka_rest_client.rb
+- lib/kafka_rest_client/avro_producer.rb
+- lib/kafka_rest_client/configuration.rb
+- lib/kafka_rest_client/errors.rb
+homepage: http://github.com/FundingCircle/kakfa_rest_client
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Ruby client for interacting with Kafka REST Proxy
+test_files: []