druiddb 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 39e6d627318a91e94886eb34b4d05b349d79153f
+  data.tar.gz: b6bc938cd4175a50e2aaa630123a03faf5119660
+SHA512:
+  metadata.gz: 202fd72edd5740aa09727d7eec183cc79ddaf2cfbd9bc529e7d908252039c78354c89ba7a58bb19f0fc550972f866d39abe9f8291fbc53bb956ec5260137a793
+  data.tar.gz: fb94c7c1e92b3a653d76851c420f8368b27d6a6dfd63402cf7bb8cca6d5e2247d028dc9fde074b0b5c943911d99e8c93bc330616fbfb487ea3691a18238fe552

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/example
+zookeeper.out
+jruby-druid.log
+.ruby-version
+*.gem

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Andre LeBlanc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1 @@
+# ruby-druid

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "irb"
+require "druiddb"
+
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/druiddb.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'druid/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "druiddb"
+  spec.version       = RubyDruid::VERSION
+  spec.authors       = ["Andre LeBlanc"]
+  spec.email         = ["andre.leblanc88@gmail.com"]
+
+  spec.summary       = 'Ruby adapter for Druid.'
+  spec.description   = 'Ruby adapter for Druid that allows reads and writes using the Tranquility Kafka API.'
+  spec.homepage      = "https://github.com/andremleblanc/ruby-druid"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activesupport", '~> 5.0'
+  spec.add_dependency "ruby-kafka", '~> 0.3'
+  spec.add_dependency "zk", '~> 1.9'
+
+  spec.add_development_dependency "bundler", '~> 1.7'
+  spec.add_development_dependency "rake", '~> 10.0'
+end

data/lib/druid/README.md

@@ -0,0 +1,20 @@
+# Druid
+This module contains all logic associated with Druid.
+
+## Node
+The `Node` classes represent Druid nodes and manage connection with them. They
+also provide the methods that are exposed natively by the Druid REST API.
+
+## Query
+The query module provides a way for the `Druid::Client` to inherit the methods
+from the `Node` classes. Additionally, the `Query` module classes provide some
+additional methods not found natively in the Druid REST API.
+
+## Writer
+The `Writer` classes utilize the Tranquility Kafka API to communicate with Druid
+nodes and allows writing.
+
+## Errors
+**Client Error:** Indicates a failure within the Ruby-Druid adapter.  
+**Connection Error:** Indicates a failed request to Druid.  
+**QueryError:** Indicates a malformed query.

data/lib/druid/client.rb

@@ -0,0 +1,22 @@
+module Druid
+  class Client
+    include Druid::Queries::Core
+    include Druid::Queries::Task
+
+    attr_reader :broker,
+                :config,
+                :coordinator,
+                :overlord,
+                :writer,
+                :zk
+
+    def initialize(options = {})
+      @config = Druid::Configuration.new(options)
+      @zk = Druid::ZK.new(config)
+      @broker = Druid::Node::Broker.new(config, zk)
+      @coordinator = Druid::Node::Coordinator.new(config, zk)
+      @overlord = Druid::Node::Overlord.new(config, zk)
+      @writer = Druid::Writer.new(config, zk)
+    end
+  end
+end

data/lib/druid/configuration.rb

@@ -0,0 +1,51 @@
+module Druid
+  class Configuration
+    DISCOVERY_PATH = '/druid/discovery'.freeze
+    INDEX_SERVICE = 'druid/overlord'.freeze
+    KAFKA_BROKER_PATH = '/brokers/ids'.freeze
+    LOG_LEVEL = :error
+    ROLLUP_GRANULARITY = :minute
+    STRONG_DELETE = false # Not recommend to be true for production.
+    TUNING_GRANULARITY = :day
+    TUNING_WINDOW = 'PT1H'.freeze
+    WAIT_TIME = 20 # Seconds
+    ZOOKEEPER = 'localhost:2181'.freeze
+
+    attr_reader :discovery_path,
+                :index_service,
+                :kafka_broker_path,
+                :log_level,
+                :rollup_granularity,
+                :strong_delete,
+                :tuning_granularity,
+                :tuning_window,
+                :wait_time,
+                :zookeeper
+
+
+    def initialize(opts = {})
+      @discovery_path = opts[:discovery_path] || DISCOVERY_PATH
+      @index_service = opts[:index_service] || INDEX_SERVICE
+      @kafka_broker_path = opts[:kafka_broker_path] || KAFKA_BROKER_PATH
+      @log_level = opts[:log_level] || LOG_LEVEL
+      @rollup_granularity = rollup_granularity_string(opts[:rollup_granularity])
+      @strong_delete = opts[:strong_delete] || STRONG_DELETE
+      @tuning_granularity = tuning_granularity_string(opts[:tuning_granularity])
+      @tuning_window = opts[:tuning_window] || TUNING_WINDOW
+      @wait_time = opts[:wait_time] || WAIT_TIME
+      @zookeeper = opts[:zookeeper] || ZOOKEEPER
+    end
+
+    private
+
+    def rollup_granularity_string(input)
+      output_string = input || ROLLUP_GRANULARITY
+      output_string.to_s.upcase.freeze
+    end
+
+    def tuning_granularity_string(input)
+      output_string = input || TUNING_GRANULARITY
+      output_string.to_s.upcase.freeze
+    end
+  end
+end

data/lib/druid/connection.rb

@@ -0,0 +1,70 @@
+# Based on: http://danknox.github.io/2013/01/27/using-rubys-native-nethttp-library/
+require 'net/http'
+
+module Druid
+  class Connection
+    CONTENT_TYPE = 'application/json'.freeze
+    VERB_MAP = {
+      :get    => ::Net::HTTP::Get,
+      :post   => ::Net::HTTP::Post,
+      :put    => ::Net::HTTP::Put,
+      :delete => ::Net::HTTP::Delete
+    }
+
+    attr_reader :http
+
+    def initialize(endpoint)
+      if endpoint.is_a? String
+        uri = URI.parse(endpoint)
+        host, port = uri.host, uri.port
+      else
+        host, port = endpoint.values_at(:host, :port)
+      end
+
+      @http = ::Net::HTTP.new(host, port)
+    end
+
+    def get(path, params = {})
+      request :get, path, params
+    end
+
+    def post(path, params = {})
+      request :post, path, params
+    end
+
+    def put(path, params = {})
+      request :put, path, params
+    end
+
+    def delete(path, params = {})
+      request :delete, path, params
+    end
+
+    private
+
+    def encode_path_params(path, params)
+      encoded = URI.encode_www_form(params)
+      [path, encoded].join("?")
+    end
+
+    def request(method, path, params)
+      case method
+      when :get
+        full_path = encode_path_params(path, params)
+        request = VERB_MAP[method].new(full_path)
+      else
+        request = VERB_MAP[method].new(path)
+        request.body = params.to_json
+      end
+
+      request.content_type = CONTENT_TYPE
+      begin
+        response = http.request(request)
+      rescue Timeout::Error, *Druid::NET_HTTP_EXCEPTIONS => e
+        raise ConnectionError, e.message
+      end
+
+      response
+    end
+  end
+end

data/lib/druid/errors.rb

@@ -0,0 +1,22 @@
+module Druid
+  class Error < StandardError; end
+  class ClientError < Error; end
+  class ConnectionError < Error; end
+  class QueryError < Error; end
+  class ValidationError < Error; end
+
+  # Adopted from: https://github.com/lostisland/faraday/blob/master/lib/faraday/adapter/net_http.rb
+  NET_HTTP_EXCEPTIONS = [
+    EOFError,
+    Errno::ECONNABORTED,
+    Errno::ECONNREFUSED,
+    Errno::ECONNRESET,
+    Errno::EHOSTUNREACH,
+    Errno::EINVAL,
+    Errno::ENETUNREACH,
+    Net::HTTPBadResponse,
+    Net::HTTPHeaderSyntaxError,
+    Net::ProtocolError,
+    SocketError
+  ]
+end

data/lib/druid/node/broker.rb

@@ -0,0 +1,35 @@
+module Druid
+  module Node
+    class Broker
+      QUERY_PATH = '/druid/v2'.freeze
+
+      attr_reader :config, :zk
+      def initialize(config, zk)
+        @config = config
+        @zk = zk
+      end
+
+      #TODO: Would caching connections be beneficial?
+      def connection
+        broker = zk.registry["#{config.discovery_path}/druid:broker"].first
+        raise Druid::ConnectionError, 'no druid brokers available' if broker.nil?
+        zk.registry["#{config.discovery_path}/druid:broker"].rotate! # round-robin load balancing
+        Druid::Connection.new(host: broker[:host], port: broker[:port])
+      end
+
+      def query(query_object)
+        begin
+          response = connection.post(QUERY_PATH, query_object)
+        rescue Druid::ConnectionError => e
+          # TODO: This sucks, make it better
+          (zk.registry["#{config.discovery_path}/druid:broker"].size - 1).times do
+            response = connection.post(QUERY_PATH, query_object)
+            break if response.code.to_i == 200
+          end
+        end
+        raise QueryError unless response.code.to_i == 200
+        JSON.parse(response.body)
+      end
+    end
+  end
+end

data/lib/druid/node/coordinator.rb

@@ -0,0 +1,117 @@
+module Druid
+  module Node
+    class Coordinator
+      DATASOURCES_PATH = '/druid/coordinator/v1/datasources/'.freeze
+
+      attr_reader :config, :zk
+      def initialize(config, zk)
+        @config = config
+        @zk = zk
+      end
+
+      # TODO: DRY; copy/paste from broker
+      def connection
+        coordinator = zk.registry["#{config.discovery_path}/druid:coordinator"].first
+        raise Druid::ConnectionError, 'no druid coordinators available' if coordinator.nil?
+        zk.registry["#{config.discovery_path}/druid:coordinator"].rotate! # round-robin load balancing
+        Druid::Connection.new(host: coordinator[:host], port: coordinator[:port])
+      end
+
+      def datasource_info(datasource_name)
+        response = connection.get(DATASOURCES_PATH + datasource_name.to_s, full: true)
+        raise ConnectionError, 'Unable to retrieve datasource information.' unless response.code.to_i == 200
+        JSON.parse(response.body)
+      end
+
+      def disable_datasource(datasource_name)
+        # response = connection.delete(DATASOURCES_PATH + datasource_name.to_s)
+        # raise ConnectionError, 'Unable to disable datasource' unless response.code.to_i == 200
+        # return true if response.code.to_i == 200
+
+        # This is a workaround for https://github.com/druid-io/druid/issues/3154
+        disable_segments(datasource_name)
+        bounded_wait_for_segments_disable(datasource_name)
+        true
+      end
+
+      # TODO: This should either be private or moved to datasource
+      def datasource_enabled?(datasource_name)
+        list_datasources.include? datasource_name
+      end
+
+      # TODO: This should either be private or moved to datasource
+      def datasource_has_segments?(datasource_name)
+        list_segments(datasource_name).any?
+      end
+
+      def disable_segment(datasource_name, segment)
+        response = connection.delete(DATASOURCES_PATH + datasource_name + '/segments/' + segment)
+        raise ConnectionError, "Unable to disable #{segment}" unless response.code.to_i == 200
+        true
+      end
+
+      # TODO: This should either be private or moved to datasource
+      def disable_segments(datasource_name)
+        segments = list_segments(datasource_name)
+        segments.each{ |segment| disable_segment(datasource_name, segment) }
+      end
+
+      def issue_kill_task(datasource_name, interval)
+        response = connection.delete(DATASOURCES_PATH + datasource_name + '/intervals/' + interval)
+        raise ConnectionError, 'Unable to issue kill task.' unless response.code.to_i == 200
+        true
+      end
+
+      def list_datasources(url_params = {})
+        response = connection.get(DATASOURCES_PATH, url_params)
+        JSON.parse(response.body) if response.code.to_i == 200
+      end
+
+      def list_segments(datasource_name)
+        response = connection.get(DATASOURCES_PATH + datasource_name + '/segments', full: true)
+        case response.code.to_i
+        when 200
+          JSON.parse(response.body).map{ |segment| segment['identifier'] }
+        when 204
+          []
+        else
+          raise ConnectionError, "Unable to list segments for #{datasource_name}"
+        end
+      end
+
+      private
+
+      def bounded_wait_for_disable(datasource_name)
+        condition = datasource_enabled?(datasource_name)
+        attempts = 0
+        max = 10
+
+        while(condition) do
+          attempts += 1
+          sleep 1
+          condition = datasource_enabled?(datasource_name)
+          break if attempts >= max
+        end
+
+        raise ClientError, 'Datasource should be disabled, but is still enabled.' unless condition
+        true
+      end
+
+      def bounded_wait_for_segments_disable(datasource_name)
+        condition = datasource_has_segments?(datasource_name)
+        attempts = 0
+        max = 60
+
+        while(condition) do
+          attempts += 1
+          sleep 1
+          condition = datasource_has_segments?(datasource_name)
+          break if attempts >= max
+        end
+
+        raise ClientError, 'Segments should be disabled, but are still enabled.' if condition
+        true
+      end
+    end
+  end
+end

data/lib/druid/node/overlord.rb

@@ -0,0 +1,60 @@
+module Druid
+  module Node
+    class Overlord
+      INDEXER_PATH = '/druid/indexer/v1/'.freeze
+      RUNNING_TASKS_PATH = (INDEXER_PATH + 'runningTasks').freeze
+      TASK_PATH = INDEXER_PATH + 'task/'
+
+      attr_reader :config, :zk
+      def initialize(config, zk)
+        @config = config
+        @zk = zk
+      end
+
+      #TODO: DRY: copy/paste
+      def connection
+        overlord = zk.registry["#{config.discovery_path}/druid:overlord"].first
+        raise Druid::ConnectionError, 'no druid overlords available' if overlord.nil?
+        zk.registry["#{config.discovery_path}/druid:overlord"].rotate! # round-robin load balancing
+        Druid::Connection.new(host: overlord[:host], port: overlord[:port])
+      end
+
+      def running_tasks(datasource_name = nil)
+        response = connection.get(RUNNING_TASKS_PATH)
+        raise ConnectionError, 'Could not retrieve running tasks' unless response.code.to_i == 200
+        tasks = JSON.parse(response.body).map{|task| task['id']}
+        tasks.select!{ |task| task.include? datasource_name } if datasource_name
+        tasks ? tasks : []
+      end
+
+      def shutdown_task(task)
+        response = connection.post(TASK_PATH + task + '/shutdown')
+        raise ConnectionError, 'Unable to shutdown task' unless response.code.to_i == 200
+        bounded_wait_for_shutdown(task)
+      end
+
+      def shutdown_tasks(datasource_name = nil)
+        tasks = running_tasks(datasource_name)
+        tasks.each{|task| shutdown_task(task)}
+      end
+
+      private
+
+      def bounded_wait_for_shutdown(task)
+        condition = !(running_tasks.include? task)
+        attempts = 0
+        max = 10
+
+        until(condition) do
+          attempts += 1
+          sleep 1
+          condition = !(running_tasks.include? task)
+          break if attempts >= max
+        end
+
+        raise ClientError, 'Task did not shutdown.' unless condition
+        true
+      end
+    end
+  end
+end

data/lib/druid/queries/core.rb

@@ -0,0 +1,11 @@
+module Druid
+  module Queries
+    module Core
+      delegate :write_point, to: :writer
+
+      def query(opts)
+        Druid::Query.create(opts.merge(broker: broker))
+      end
+    end
+  end
+end

data/lib/druid/queries/task.rb

@@ -0,0 +1,7 @@
+module Druid
+  module Queries
+    module Task
+      delegate :shutdown_tasks, to: :overlord
+    end
+  end
+end

data/lib/druid/query.rb

@@ -0,0 +1,182 @@
+module Druid
+  class Query
+    attr_reader :aggregations,
+                :broker,
+                :dimensions,
+                :end_interval,
+                :fill_value,
+                :granularity,
+                :query_opts,
+                :query_type,
+                :range,
+                :result_key,
+                :start_interval
+
+    def initialize(opts)
+      @aggregations = opts[:aggregations].map{|agg| agg[:name]}
+      @broker = opts[:broker]
+      @dimensions = opts[:dimensions]
+      @fill_value = opts[:fill_value]
+      @granularity = opts[:granularity]
+      @range = parse_range(opts[:intervals])
+      @query_type = opts[:queryType]
+      @end_interval = calculate_end_interval
+      @start_interval = calculate_start_interval
+      @query_opts = opts_for_query(opts)
+    end
+
+    def execute
+      result = broker.query(query_opts)
+      fill_query_results(result)
+    end
+
+    private
+
+    # TODO: Can this be made smarter? Prefer to avoid case statements.
+    # Cases found here: http://druid.io/docs/latest/querying/granularities.html
+    def advance_interval(time)
+      case granularity
+      when 'second'
+        time.advance(seconds: 1)
+      when 'minute'
+        time.advance(minutes: 1)
+      when 'fifteen_minute'
+        time.advance(minutes: 15)
+      when 'thirty_minute'
+        time.advance(minutes: 30)
+      when 'hour'
+        time.advance(hours: 1)
+      when 'day'
+        time.advance(days: 1)
+      when 'week'
+        time.advance(weeks: 1)
+      when 'month'
+        time.advance(months: 1)
+      when 'quarter'
+        time.advance(months: 3)
+      when 'year'
+        time.advance(years: 1)
+      else
+        raise Druid::QueryError, 'Unsupported granularity'
+      end
+    end
+
+    def calculate_end_interval
+      iso8601_duration_end_interval(range)
+    end
+
+    def calculate_start_interval
+      time = iso8601_duration_start_interval(range)
+      start_of_interval(time)
+    end
+
+    def fill_empty_intervals(points, opts = {})
+      interval = start_interval
+      result = []
+
+      while interval <= end_interval do
+        # TODO:
+        # This will search the points every time, could be more performant if
+        # we track the 'current point' in the points and only compare the
+        # current point's timestamp
+        point = find_or_create_point(interval, points)
+        aggregations.each do |aggregation|
+          point[result_key][aggregation] = fill_value if point[result_key][aggregation].blank?
+          point[result_key].merge!(opts)
+        end
+        result << point
+        interval = advance_interval(interval)
+      end
+
+      result
+    end
+
+    # NOTE:
+    # This responsibility really lies in Druid, but until the feature works
+    # reliably in Druid, this is serves the purpose.
+    # https://github.com/druid-io/druid/issues/2106
+    def fill_query_results(query_result)
+      return query_result unless query_result.present? && fill_value.present?
+      parse_result_key(query_result.first)
+
+      #TODO: handle multi-dimensional group by
+      if group_by?
+        result = []
+        dimension_key = dimensions.first
+        groups = query_result.group_by{ |point| point[result_key][dimension_key] }
+        groups.each do |dimension_value, dimension_points|
+          result += fill_empty_intervals(dimension_points, { dimension_key => dimension_value })
+        end
+        result
+      else
+        fill_empty_intervals(query_result)
+      end
+    end
+
+    def find_or_create_point(interval, points)
+      point = points.find{ |point| point['timestamp'].to_s.to_time == interval.to_time }
+      point.present? ? point : { 'timestamp' => interval.iso8601(3), result_key => {} }
+    end
+
+    def group_by?
+      query_type == 'groupBy'
+    end
+
+    def iso8601_duration_start_interval(duration)
+      duration.split('/').first.to_time.utc
+    end
+
+    def iso8601_duration_end_interval(duration)
+      duration.split('/').last.to_time.utc
+    end
+
+    def opts_for_query(opts)
+      opts.except(:fill_value, :broker)
+    end
+
+    def parse_range(range)
+      range.is_a?(Array) ? range.first : range
+    end
+
+    def parse_result_key(point)
+      @result_key = point['event'].present? ? 'event' : 'result'
+    end
+
+    # TODO: Can this be made smarter? Prefer to avoid case statements.
+    # Cases found here: http://druid.io/docs/latest/querying/granularities.html
+    def start_of_interval(time)
+      case granularity
+      when 'second'
+        time.change(usec: 0)
+      when 'minute'
+        time.beginning_of_minute
+      when 'fifteen_minute'
+        first_fifteen = [45, 30, 15, 0].detect{ |m| m <= time.min }
+        time.change(min: first_fifteen)
+      when 'thirty_minute'
+        first_thirty = [30, 0].detect{ |m| m <= time.min }
+        time.change(min: first_thirty)
+      when 'hour'
+        time.beginning_of_hour
+      when 'day'
+        time.beginning_of_day
+      when 'week'
+        time.beginning_of_week
+      when 'month'
+        time.beginning_of_month
+      when 'quarter'
+        time.beginning_of_quarter
+      when 'year'
+        time.beginning_of_year
+      else
+        time
+      end
+    end
+
+    class << self
+      def create(opts)
+        new(opts).execute
+      end
+    end
+  end
+end

data/lib/druid/version.rb

@@ -0,0 +1,3 @@
+module RubyDruid
+	VERSION = '1.0.0'
+end

data/lib/druid/writer.rb

@@ -0,0 +1,47 @@
+#TODO: Seems to be a delay after shutting down Kafka and ZK updating
+module Druid
+  class Writer
+    attr_reader :config, :producer, :zk
+    def initialize(config, zk)
+      @config = config
+      @zk = zk
+      init_producer
+      zk.register_listener(self, :handle_kafka_state_change)
+    end
+
+    def write_point(datasource, datapoint)
+      raise Druid::ConnectionError, 'no kafka brokers available' if producer.nil?
+      producer.produce(datapoint, topic: datasource)
+    end
+
+    private
+
+    def broker_list
+      zk.registry["/brokers/ids"].map{|instance| "#{instance[:host]}:#{instance[:port]}" }.join(',')
+    end
+
+    def handle_kafka_state_change(service)
+      if service == config.kafka_broker_path
+        producer.shutdown
+        init_producer
+      end
+    end
+
+    def init_producer
+      producer_options = {
+        seed_brokers: broker_list,
+        client_id: "ruby-druid"
+      }
+
+      if broker_list.present?
+        kafka = Kafka.new(producer_options)
+        producer = kafka.async_producer(delivery_threshold: 100, delivery_interval: 10)
+        producer.deliver_messages
+      else
+        producer = nil
+      end
+
+      @producer = producer
+    end
+  end
+end

data/lib/druid/zk.rb

@@ -0,0 +1,63 @@
+module Druid
+  class ZK
+    attr_accessor :registry
+    attr_reader :client, :config, :listeners
+
+    #TODO: Test and handle ZK partitions
+    def initialize(config)
+      @client = ::ZK.new(config.zookeeper)
+      @config = config
+      @listeners = []
+      @registry = {}
+      register
+    end
+
+    def register_listener(object, method)
+      listeners << ->(*args) { object.send(method, *args) }
+    end
+
+    private
+
+    def announce(service)
+      # puts "announcing #{service}"
+      listeners.each { |listener| listener.call(service) }
+    end
+
+    def register
+      register_service("#{config.discovery_path}/druid:broker")
+      register_service("#{config.discovery_path}/druid:coordinator")
+      register_service("#{config.discovery_path}/druid:overlord")
+      register_service("#{config.kafka_broker_path}")
+    end
+
+    def register_service(service)
+      # puts "registering #{service}"
+      #TODO: Thead safety, lock this registry key
+      subscribe_to_service(service)
+      renew_service_instances(service)
+    end
+
+    def renew_service_instances(service)
+      # puts "activating registered subscriptions on #{service}"
+      instances = client.children(service, watch: true)
+
+      # puts "emptying #{service} from registry"
+      registry[service] = []
+      instances.each do |instance|
+        data = JSON.parse(client.get("#{service}/#{instance}").first)
+        host = data['address'] || data['host']
+        port = data['port']
+        # puts "adding #{host}:#{port} to registry for #{service}"
+        registry[service] << { host: host, port: port }
+      end
+    end
+
+    def subscribe_to_service(service)
+      subscription = client.register(service) do |event|
+        # puts "watched event for #{service} detected"
+        renew_service_instances(event.path)
+        announce(event.path)
+      end
+    end
+  end
+end

data/lib/druiddb.rb

@@ -0,0 +1,21 @@
+require "active_support/all"
+require "ruby-kafka"
+require "json"
+require "zk"
+
+require "druid/configuration"
+require "druid/connection"
+require "druid/errors"
+require "druid/query"
+require "druid/version"
+require "druid/zk"
+
+require "druid/node/broker"
+require "druid/node/coordinator"
+require "druid/node/overlord"
+
+require "druid/queries/core"
+require "druid/queries/task"
+
+require "druid/writer"
+require "druid/client"

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: druiddb
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Andre LeBlanc
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-07-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: ruby-kafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: zk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Ruby adapter for Druid that allows reads and writes using the Tranquility
+  Kafka API.
+email:
+- andre.leblanc88@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- druiddb.gemspec
+- lib/druid/README.md
+- lib/druid/client.rb
+- lib/druid/configuration.rb
+- lib/druid/connection.rb
+- lib/druid/errors.rb
+- lib/druid/node/broker.rb
+- lib/druid/node/coordinator.rb
+- lib/druid/node/overlord.rb
+- lib/druid/queries/core.rb
+- lib/druid/queries/task.rb
+- lib/druid/query.rb
+- lib/druid/version.rb
+- lib/druid/writer.rb
+- lib/druid/zk.rb
+- lib/druiddb.rb
+homepage: https://github.com/andremleblanc/ruby-druid
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: Ruby adapter for Druid.
+test_files: []