druiddb 1.0.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77f96072ce4ca16fd1d5f9d4ad147f646da2e6a8
-  data.tar.gz: 1c17a5ea1d268cc3ba0a72aa1acdffd21054f26e
+  metadata.gz: 6aff717c3c1644264319311423dd20b93c6b1c1b
+  data.tar.gz: 13ee2b193c9fd2eb5ce6c875176b950f4be736aa
 SHA512:
-  metadata.gz: 43d399682a94461de6ab511252a3807ec7610f0393c36be70dfe86ecf42e5d45f172175c3b4551cb3c954ac2bbdfddc3d2d0c25105e00f0b4b426f939707be1a
-  data.tar.gz: 8a2c36065fe408b8c092946ad69b003cde7f8ace3562ba12d8efc61378b247bd16a65e1fcb520db09e9e265accb7c1d171fd069a36b85358c2a504f1d4d4895f
+  metadata.gz: c20d56f9c873ded1c52c99ee7fe3e6e6517c652335d45c8cf8e3696063b25e1291eff03a210036c0b3128d76ae6e86303f90e0d62ad5ede3ded5fa7a72db4ee2
+  data.tar.gz: 1c8b7c392900c2b99c517186a498ed84d9843ccb40ba95ff24e0b929a9448e58b387fc8d20257debbbb69bdead3d5b0dbc7c1ab059feef65a63c6b0199047269

data/.gitignore

@@ -9,6 +9,5 @@
 /tmp/
 /example
 zookeeper.out
-jruby-druid.log
 .ruby-version
 *.gem

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,6 @@
+inherit_from: .rubocop_todo.yml
+
+Documentation:
+  Enabled: false
+Metrics/LineLength:
+  Max: 100

data/.rubocop_todo.yml

@@ -0,0 +1,35 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2017-08-20 22:13:21 -0400 using RuboCop version 0.49.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 11
+Metrics/AbcSize:
+  Max: 22
+
+# Offense count: 1
+# Configuration parameters: CountComments.
+Metrics/ClassLength:
+  Max: 145
+
+# Offense count: 3
+Metrics/CyclomaticComplexity:
+  Max: 11
+
+# Offense count: 2
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Metrics/LineLength:
+  Max: 108
+
+# Offense count: 9
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 26
+
+# Offense count: 1
+Metrics/PerceivedComplexity:
+  Max: 10

data/.travis.yml

@@ -0,0 +1,15 @@
+language: ruby
+sudo: required
+
+services:
+  - docker
+
+before_script:
+  - docker-compose up -d
+  - docker build -t druiddb-ruby .
+
+script:
+  - docker run -it --network=druiddbruby_druiddb druiddb-ruby bin/run_tests.sh
+
+after_script:
+  - docker-compose down

data/Dockerfile

@@ -0,0 +1,20 @@
+FROM ruby:2.2.6
+MAINTAINER Andre LeBlanc <andre.leblanc88@gmail.com>
+
+RUN apt-get update
+
+WORKDIR /druiddb-ruby
+
+COPY lib/druiddb/version.rb lib/druiddb/version.rb
+COPY druiddb.gemspec druiddb.gemspec
+COPY Gemfile Gemfile
+
+RUN git init
+RUN bundle install
+
+COPY bin bin
+COPY lib lib
+COPY spec spec
+COPY Rakefile Rakefile
+
+CMD bin/console

data/README.md

@@ -1 +1,130 @@
-# ruby-druid
+# druiddb-ruby
+
+[![Build Status](https://travis-ci.org/andremleblanc/druiddb-ruby.svg?branch=master)](https://travis-ci.org/andremleblanc/druiddb-ruby)
+[![Gem Version](https://badge.fury.io/rb/druiddb.svg)](https://badge.fury.io/rb/druiddb)
+[![Code Climate](https://codeclimate.com/github/andremleblanc/druiddb-ruby/badges/gpa.svg)](https://codeclimate.com/github/andremleblanc/druiddb-ruby)
+[![Test Coverage](https://codeclimate.com/github/andremleblanc/druiddb-ruby/badges/coverage.svg)](https://codeclimate.com/github/andremleblanc/druiddb-ruby/coverage)
+[![Dependency Status](https://gemnasium.com/badges/github.com/andremleblanc/druiddb-ruby.svg)](https://gemnasium.com/github.com/andremleblanc/druiddb-ruby)
+
+This documentation is intended to be a quick-start guide, not a comprehensive
+list of all available methods and configuration options. Please look through
+the source for more information; a great place to get started is `DruidDB::Client`
+and the `DruidDB::Query` modules as they expose most of the methods on the client.
+
+This guide assumes a significant knowledge of Druid, for more info:
+http://druid.io/docs/latest/design/index.html
+
+## Install
+
+```bash
+$ gem install druiddb
+```
+
+## Usage
+
+### Creating a Client
+```ruby
+client = DruidDB::Client.new()
+```
+*Note:* There are many configuration options, please take a look at
+`DruidDB::Configuration` for more details.
+
+### Writing Data
+
+#### Kafka Indexing service
+This gem leverages the [Kafka Indexing Service](http://druid.io/docs/latest/development/extensions-core/kafka-ingestion.html) for ingesting data. The gem pushes datapoints onto Kafka topics (typically named after the datasource). You can also use the gem to upload an ingestion spec, which is needed for Druid to consume the Kafka topic.
+
+This repo contains a `docker-compose.yml` build that may help bootstrap development with Druid and the Kafka Indexing Service. It's what we use for integration testing.
+
+#### Submitting an Ingestion Spec
+
+```ruby
+path = 'path/to/spec.json'
+client.submit_supervisor_spec(path)
+```
+
+####  Writing Datapoints
+```ruby
+topic_name = 'foo'
+datapoint = {
+  timestamp: Time.now.utc.iso8601,
+  foo: 'bar',
+  units: 1
+}
+client.write_point(topic_name, datapoint)
+```
+
+### Reading Data
+
+#### Querying
+```ruby
+client.query(
+  queryType: 'timeseries',
+  dataSource: 'foo',
+  granularity: 'day',
+  intervals: Time.now.utc.advance(days: -30) + '/' + Time.now.utc.iso8601,
+  aggregations: [{ type: 'longSum', name: 'baz', fieldName: 'baz' }]
+)
+```
+The `query` method POSTs the query to Druid; for information on
+querying Druid: http://druid.io/docs/latest/querying/querying.html. This is
+intentionally simple to allow all current features and hopefully all future
+features of the Druid query language without updating the gem.
+
+##### Fill Empty Intervals
+
+Currently, Druid will not fill empty intervals for which there are no points. To
+accommodate this need until it is handled more efficiently in Druid, use the
+experimental `fill_value` feature in your query. This ensure you get a result
+for every interval in intervals.
+
+This has only been tested with 'timeseries' and single-dimension 'groupBy'
+queries with simple granularities.
+
+```ruby
+client.query(
+  queryType: 'timeseries',
+  dataSource: 'foo',
+  granularity: 'day',
+  intervals: Time.now.utc.advance(days: -30) + '/' + Time.now.utc.iso8601,
+  aggregations: [{ type: 'longSum', name: 'baz', fieldName: 'baz' }],
+  fill_value: 0
+)
+```
+
+### Management
+List datasources.
+```ruby
+client.list_datasources
+```
+
+List supervisor tasks.
+```ruby
+client.supervisor_tasks
+```
+
+## Development
+
+### Docker Compose
+This project uses docker-compose to provide a development environment.
+
+1. git clone the project
+2. cd into project
+3. `docker-compose up` - this will download necessary images and run all dependencies in the foreground.
+
+Then you can use `docker build -t some_tag .` to build the Docker image for this project after making changes and `docker run -it --network=druiddbruby_druiddb some_tag some_command` to interact with it.
+
+### Metabase
+
+Viewing data in the database can be a bit annoying, use a tool like [Metabase](https://github.com/metabase/metabase) makes this much easier and is what I personally do when developing.
+
+## Testing
+
+Testing is run utilizing the docker-compose environment.
+
+1. `docker-compose up`
+2. `docker run -it --network=druiddbruby_druiddb druiddb-ruby bin/run_tests.sh`
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -1,6 +1,13 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'druiddb'
 
-RSpec::Core::RakeTask.new(:spec)
-
-task :default => :spec
+namespace :db do
+  namespace :test do
+    task :prepare do
+      client = DruidDB::Client.new(zookeeper: 'zookeeper:2181')
+      client.submit_supervisor_spec("#{Dir.pwd}/spec/ingestion_specs/xwings_spec.json")
+      puts client.supervisor_tasks
+    end
+  end
+end

data/bin/console

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
-require "bundler/setup"
-require "irb"
-require "druiddb"
+require 'bundler/setup'
+require 'irb'
+require 'druiddb'
 
 IRB.start

data/bin/run_tests.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+spec/wait-for-it.sh overlord:8090 --timeout=30 --strict -- rspec

data/docker-compose.yml

@@ -0,0 +1,100 @@
+version: "3"
+
+networks:
+  druiddb:
+
+volumes:
+  druid_fs:
+
+services:
+  zookeeper:
+    image: zookeeper:3.4
+    networks:
+      - druiddb
+    ports:
+      - '2181:2181'
+
+  derby:
+    image: adito/apache-derby
+    networks:
+      - druiddb
+
+  kafka:
+    image: wurstmeister/kafka:0.10.2.1
+    networks:
+      - druiddb
+    ports:
+      - '7203:7203'
+      - '9092:9092'
+    depends_on:
+      - zookeeper
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: kafka
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+
+  broker:
+    image: andremleblanc/druid-broker:0.9.2
+    networks:
+      - druiddb
+    ports:
+      - '8082:8082'
+    depends_on:
+      - zookeeper
+      - derby
+      - kafka
+    volumes:
+      - druid_fs:/druid-0.9.2/var/druid/
+
+  coordinator:
+    image: andremleblanc/druid-coordinator:0.9.2
+    networks:
+      - druiddb
+    ports:
+      - '8081:8081'
+    depends_on:
+      - zookeeper
+      - derby
+      - kafka
+    volumes:
+      - druid_fs:/druid-0.9.2/var/druid/
+
+  historical:
+    image: andremleblanc/druid-historical:0.9.2
+    networks:
+      - druiddb
+    ports:
+      - '8083:8083'
+    depends_on:
+      - zookeeper
+      - derby
+      - kafka
+    volumes:
+      - druid_fs:/druid-0.9.2/var/druid/
+
+  middlemanager:
+    image: andremleblanc/druid-middlemanager:0.9.2
+    networks:
+      - druiddb
+    ports:
+      - '8091:8091'
+    depends_on:
+      - zookeeper
+      - derby
+      - kafka
+    volumes:
+      - druid_fs:/druid-0.9.2/var/druid/
+
+  overlord:
+    image: andremleblanc/druid-overlord:0.9.2
+    networks:
+      - druiddb
+    ports:
+      - '8090:8090'
+    depends_on:
+      - zookeeper
+      - kafka
+      - derby
+    volumes:
+      - druid_fs:/druid-0.9.2/var/druid/

data/druiddb.gemspec

@@ -1,28 +1,32 @@
 # coding: utf-8
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'druid/version'
+require 'druiddb/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "druiddb"
-  spec.version       = Druiddb::VERSION
-  spec.authors       = ["Andre LeBlanc"]
-  spec.email         = ["andre.leblanc88@gmail.com"]
+  spec.name          = 'druiddb'
+  spec.version       = DruidDB::VERSION
+  spec.authors       = ['Andre LeBlanc']
+  spec.email         = ['andre.leblanc88@gmail.com']
 
-  spec.summary       = 'Ruby adapter for Druid.'
-  spec.description   = 'Ruby adapter for Druid that allows reads and writes using the Tranquility Kafka API.'
-  spec.homepage      = "https://github.com/andremleblanc/druiddb"
-  spec.license       = "MIT"
+  spec.summary       = 'Ruby client for Druid.'
+  spec.description   = 'Ruby client for reading from and writing to Druid.'
+  spec.homepage      = 'https://github.com/andremleblanc/druiddb-ruby'
+  spec.license       = 'MIT'
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_dependency "activesupport", '>= 4.0'
-  spec.add_dependency "ruby-kafka", '~> 0.3'
-  spec.add_dependency "zk", '~> 1.9'
+  spec.add_dependency 'activesupport', '> 4.0'
+  spec.add_dependency 'ruby-kafka', '~> 0.3'
+  spec.add_dependency 'zk', '~> 1.9'
 
-  spec.add_development_dependency "bundler", '~> 1.7'
-  spec.add_development_dependency "rake", '~> 10.0'
+  spec.add_development_dependency 'bundler', '~> 1.7'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.6'
 end

data/lib/druiddb.rb

@@ -1,21 +1,22 @@
-require "active_support/all"
-require "ruby-kafka"
-require "json"
-require "zk"
+require 'active_support/all'
+require 'ruby-kafka'
+require 'json'
+require 'zk'
 
-require "druid/configuration"
-require "druid/connection"
-require "druid/errors"
-require "druid/query"
-require "druid/version"
-require "druid/zk"
+require 'druiddb/configuration'
+require 'druiddb/connection'
+require 'druiddb/errors'
+require 'druiddb/query'
+require 'druiddb/version'
+require 'druiddb/zk'
 
-require "druid/node/broker"
-require "druid/node/coordinator"
-require "druid/node/overlord"
+require 'druiddb/node/broker'
+require 'druiddb/node/coordinator'
+require 'druiddb/node/overlord'
 
-require "druid/queries/core"
-require "druid/queries/task"
+require 'druiddb/queries/core'
+require 'druiddb/queries/datasources'
+require 'druiddb/queries/task'
 
-require "druid/writer"
-require "druid/client"
+require 'druiddb/writer'
+require 'druiddb/client'

data/lib/druiddb/client.rb

@@ -0,0 +1,23 @@
+module DruidDB
+  class Client
+    include DruidDB::Queries::Core
+    include DruidDB::Queries::Datasources
+    include DruidDB::Queries::Task
+
+    attr_reader :broker,
+                :config,
+                :coordinator,
+                :overlord,
+                :writer,
+                :zk
+
+    def initialize(options = {})
+      @config = DruidDB::Configuration.new(options)
+      @zk = DruidDB::ZK.new(config)
+      @broker = DruidDB::Node::Broker.new(config, zk)
+      @coordinator = DruidDB::Node::Coordinator.new(config, zk)
+      @overlord = DruidDB::Node::Overlord.new(config, zk)
+      @writer = DruidDB::Writer.new(config, zk)
+    end
+  end
+end

data/lib/{druid → druiddb}/configuration.rb

@@ -1,17 +1,19 @@
-module Druid
+module DruidDB
   class Configuration
+    CLIENT_ID = 'druiddb-ruby'.freeze
     DISCOVERY_PATH = '/druid/discovery'.freeze
     INDEX_SERVICE = 'druid/overlord'.freeze
     KAFKA_BROKER_PATH = '/brokers/ids'.freeze
     LOG_LEVEL = :error
     ROLLUP_GRANULARITY = :minute
-    STRONG_DELETE = false # Not recommend to be true for production.
+    STRONG_DELETE = false
     TUNING_GRANULARITY = :day
     TUNING_WINDOW = 'PT1H'.freeze
-    WAIT_TIME = 20 # Seconds
+    WAIT_TIME = 20
     ZOOKEEPER = 'localhost:2181'.freeze
 
-    attr_reader :discovery_path,
+    attr_reader :client_id,
+                :discovery_path,
                 :index_service,
                 :kafka_broker_path,
                 :log_level,
@@ -22,8 +24,8 @@ module Druid
                 :wait_time,
                 :zookeeper
 
-
     def initialize(opts = {})
+      @client_id = opts[:client_id] || CLIENT_ID
       @discovery_path = opts[:discovery_path] || DISCOVERY_PATH
       @index_service = opts[:index_service] || INDEX_SERVICE
       @kafka_broker_path = opts[:kafka_broker_path] || KAFKA_BROKER_PATH

data/lib/{druid → druiddb}/connection.rb

@@ -1,22 +1,23 @@
 # Based on: http://danknox.github.io/2013/01/27/using-rubys-native-nethttp-library/
 require 'net/http'
 
-module Druid
+module DruidDB
   class Connection
     CONTENT_TYPE = 'application/json'.freeze
     VERB_MAP = {
-      :get    => ::Net::HTTP::Get,
-      :post   => ::Net::HTTP::Post,
-      :put    => ::Net::HTTP::Put,
-      :delete => ::Net::HTTP::Delete
-    }
+      get: ::Net::HTTP::Get,
+      post: ::Net::HTTP::Post,
+      put: ::Net::HTTP::Put,
+      delete: ::Net::HTTP::Delete
+    }.freeze
 
     attr_reader :http
 
     def initialize(endpoint)
       if endpoint.is_a? String
         uri = URI.parse(endpoint)
-        host, port = uri.host, uri.port
+        host = uri.host
+        port = uri.port
       else
         host, port = endpoint.values_at(:host, :port)
       end
@@ -44,7 +45,7 @@ module Druid
 
     def encode_path_params(path, params)
       encoded = URI.encode_www_form(params)
-      [path, encoded].join("?")
+      [path, encoded].join('?')
     end
 
     def request(method, path, params)
@@ -60,7 +61,7 @@ module Druid
       request.content_type = CONTENT_TYPE
       begin
         response = http.request(request)
-      rescue Timeout::Error, *Druid::NET_HTTP_EXCEPTIONS => e
+      rescue Timeout::Error, *DruidDB::NET_HTTP_EXCEPTIONS => e
         raise ConnectionError, e.message
       end
 

data/lib/{druid → druiddb}/errors.rb

@@ -1,4 +1,4 @@
-module Druid
+module DruidDB
   class Error < StandardError; end
   class ClientError < Error; end
   class ConnectionError < Error; end
@@ -18,5 +18,5 @@ module Druid
     Net::HTTPHeaderSyntaxError,
     Net::ProtocolError,
     SocketError
-  ]
+  ].freeze
 end

data/lib/{druid → druiddb}/node/broker.rb

@@ -1,4 +1,4 @@
-module Druid
+module DruidDB
   module Node
     class Broker
       QUERY_PATH = '/druid/v2'.freeze
@@ -9,18 +9,18 @@ module Druid
         @zk = zk
       end
 
-      #TODO: Would caching connections be beneficial?
       def connection
         broker = zk.registry["#{config.discovery_path}/druid:broker"].first
-        raise Druid::ConnectionError, 'no druid brokers available' if broker.nil?
+        raise DruidDB::ConnectionError, 'no druid brokers available' if broker.nil?
         zk.registry["#{config.discovery_path}/druid:broker"].rotate! # round-robin load balancing
-        Druid::Connection.new(host: broker[:host], port: broker[:port])
+        DruidDB::Connection.new(host: broker[:host], port: broker[:port])
       end
 
       def query(query_object)
         begin
           response = connection.post(QUERY_PATH, query_object)
-        rescue Druid::ConnectionError => e
+        rescue DruidDB::ConnectionError
+          # TODO: Log
           # TODO: This sucks, make it better
           (zk.registry["#{config.discovery_path}/druid:broker"].size - 1).times do
             response = connection.post(QUERY_PATH, query_object)

data/lib/{druid → druiddb}/node/coordinator.rb

@@ -1,4 +1,4 @@
-module Druid
+module DruidDB
   module Node
     class Coordinator
       DATASOURCES_PATH = '/druid/coordinator/v1/datasources/'.freeze
@@ -12,14 +12,17 @@ module Druid
       # TODO: DRY; copy/paste from broker
       def connection
         coordinator = zk.registry["#{config.discovery_path}/druid:coordinator"].first
-        raise Druid::ConnectionError, 'no druid coordinators available' if coordinator.nil?
-        zk.registry["#{config.discovery_path}/druid:coordinator"].rotate! # round-robin load balancing
-        Druid::Connection.new(host: coordinator[:host], port: coordinator[:port])
+        raise DruidDB::ConnectionError, 'no druid coordinators available' if coordinator.nil?
+        # round-robin load balancing
+        zk.registry["#{config.discovery_path}/druid:coordinator"].rotate!
+        DruidDB::Connection.new(host: coordinator[:host], port: coordinator[:port])
       end
 
       def datasource_info(datasource_name)
         response = connection.get(DATASOURCES_PATH + datasource_name.to_s, full: true)
-        raise ConnectionError, 'Unable to retrieve datasource information.' unless response.code.to_i == 200
+        unless response.code.to_i == 200
+          raise ConnectionError, 'Unable to retrieve datasource information.'
+        end
         JSON.parse(response.body)
       end
 
@@ -53,7 +56,7 @@ module Druid
       # TODO: This should either be private or moved to datasource
       def disable_segments(datasource_name)
         segments = list_segments(datasource_name)
-        segments.each{ |segment| disable_segment(datasource_name, segment) }
+        segments.each { |segment| disable_segment(datasource_name, segment) }
       end
 
       def issue_kill_task(datasource_name, interval)
@@ -71,7 +74,7 @@ module Druid
         response = connection.get(DATASOURCES_PATH + datasource_name + '/segments', full: true)
         case response.code.to_i
         when 200
-          JSON.parse(response.body).map{ |segment| segment['identifier'] }
+          JSON.parse(response.body).map { |segment| segment['identifier'] }
         when 204
           []
         else
@@ -86,7 +89,7 @@ module Druid
         attempts = 0
         max = 10
 
-        while(condition) do
+        while condition
           attempts += 1
           sleep 1
           condition = datasource_enabled?(datasource_name)
@@ -102,7 +105,7 @@ module Druid
         attempts = 0
         max = 60
 
-        while(condition) do
+        while condition
           attempts += 1
           sleep 1
           condition = datasource_has_segments?(datasource_name)

data/lib/{druid → druiddb}/node/overlord.rb

@@ -1,9 +1,10 @@
-module Druid
+module DruidDB
   module Node
     class Overlord
       INDEXER_PATH = '/druid/indexer/v1/'.freeze
       RUNNING_TASKS_PATH = (INDEXER_PATH + 'runningTasks').freeze
-      TASK_PATH = INDEXER_PATH + 'task/'
+      TASK_PATH = (INDEXER_PATH + 'task/').freeze
+      SUPERVISOR_PATH = (INDEXER_PATH + 'supervisor/').freeze
 
       attr_reader :config, :zk
       def initialize(config, zk)
@@ -11,19 +12,19 @@ module Druid
         @zk = zk
       end
 
-      #TODO: DRY: copy/paste
+      # TODO: DRY: copy/paste
       def connection
         overlord = zk.registry["#{config.discovery_path}/druid:overlord"].first
-        raise Druid::ConnectionError, 'no druid overlords available' if overlord.nil?
+        raise DruidDB::ConnectionError, 'no druid overlords available' if overlord.nil?
         zk.registry["#{config.discovery_path}/druid:overlord"].rotate! # round-robin load balancing
-        Druid::Connection.new(host: overlord[:host], port: overlord[:port])
+        DruidDB::Connection.new(host: overlord[:host], port: overlord[:port])
       end
 
       def running_tasks(datasource_name = nil)
         response = connection.get(RUNNING_TASKS_PATH)
         raise ConnectionError, 'Could not retrieve running tasks' unless response.code.to_i == 200
-        tasks = JSON.parse(response.body).map{|task| task['id']}
-        tasks.select!{ |task| task.include? datasource_name } if datasource_name
+        tasks = JSON.parse(response.body).map { |task| task['id'] }
+        tasks.select! { |task| task.include? datasource_name } if datasource_name
         tasks ? tasks : []
       end
 
@@ -35,7 +36,20 @@ module Druid
 
       def shutdown_tasks(datasource_name = nil)
         tasks = running_tasks(datasource_name)
-        tasks.each{|task| shutdown_task(task)}
+        tasks.each { |task| shutdown_task(task) }
+      end
+
+      def supervisor_tasks
+        response = connection.get(SUPERVISOR_PATH)
+        raise ConnectionError, 'Could not retrieve supervisors' unless response.code.to_i == 200
+        JSON.parse(response.body)
+      end
+
+      def submit_supervisor_spec(filepath)
+        spec = JSON.parse(File.read(filepath))
+        response = connection.post(SUPERVISOR_PATH, spec)
+        raise ConnectionError, 'Unable to submit spec' unless response.code.to_i == 200
+        JSON.parse(response.body)
       end
 
       private
@@ -45,7 +59,7 @@ module Druid
         attempts = 0
         max = 10
 
-        until(condition) do
+        until condition
           attempts += 1
           sleep 1
           condition = !(running_tasks.include? task)

data/lib/{druid → druiddb}/queries/core.rb

@@ -1,10 +1,10 @@
-module Druid
+module DruidDB
   module Queries
     module Core
       delegate :write_point, to: :writer
 
       def query(opts)
-        Druid::Query.create(opts.merge(broker: broker))
+        DruidDB::Query.create(opts.merge(broker: broker))
       end
     end
   end

data/lib/druiddb/queries/datasources.rb

@@ -0,0 +1,7 @@
+module DruidDB
+  module Queries
+    module Datasources
+      delegate :list_datasources, to: :coordinator
+    end
+  end
+end

data/lib/druiddb/queries/task.rb

@@ -0,0 +1,10 @@
+module DruidDB
+  module Queries
+    module Task
+      delegate :shutdown_tasks,
+               :supervisor_tasks,
+               :submit_supervisor_spec,
+               to: :overlord
+    end
+  end
+end

data/lib/{druid → druiddb}/query.rb

@@ -1,4 +1,4 @@
-module Druid
+module DruidDB
   class Query
     attr_reader :aggregations,
                 :broker,
@@ -13,7 +13,7 @@ module Druid
                 :start_interval
 
     def initialize(opts)
-      @aggregations = opts[:aggregations].map{|agg| agg[:name]}
+      @aggregations = opts[:aggregations].map { |agg| agg[:name] }
       @broker = opts[:broker]
       @dimensions = opts[:dimensions]
       @fill_value = opts[:fill_value]
@@ -57,7 +57,7 @@ module Druid
       when 'year'
         time.advance(years: 1)
       else
-        raise Druid::QueryError, 'Unsupported granularity'
+        raise DruidDB::QueryError, 'Unsupported granularity'
       end
     end
 
@@ -74,9 +74,8 @@ module Druid
       interval = start_interval
       result = []
 
-      while interval <= end_interval do
-        # TODO:
-        # This will search the points every time, could be more performant if
+      while interval <= end_interval
+        # TODO: This will search the points every time, could be more performant if
         # we track the 'current point' in the points and only compare the
         # current point's timestamp
         point = find_or_create_point(interval, points)
@@ -99,13 +98,13 @@ module Druid
       return query_result unless query_result.present? && fill_value.present?
       parse_result_key(query_result.first)
 
-      #TODO: handle multi-dimensional group by
+      # TODO: handle multi-dimensional group by
       if group_by?
         result = []
         dimension_key = dimensions.first
-        groups = query_result.group_by{ |point| point[result_key][dimension_key] }
+        groups = query_result.group_by { |point| point[result_key][dimension_key] }
         groups.each do |dimension_value, dimension_points|
-          result += fill_empty_intervals(dimension_points, { dimension_key => dimension_value })
+          result += fill_empty_intervals(dimension_points, dimension_key => dimension_value)
         end
         result
       else
@@ -114,7 +113,7 @@ module Druid
     end
 
     def find_or_create_point(interval, points)
-      point = points.find{ |point| point['timestamp'].to_s.to_time == interval.to_time }
+      point = points.find { |p| p['timestamp'].to_s.to_time == interval.to_time }
       point.present? ? point : { 'timestamp' => interval.iso8601(3), result_key => {} }
     end
 
@@ -151,10 +150,10 @@ module Druid
       when 'minute'
         time.beginning_of_minute
       when 'fifteen_minute'
-        first_fifteen = [45, 30, 15, 0].detect{ |m| m <= time.min }
+        first_fifteen = [45, 30, 15, 0].detect { |m| m <= time.min }
         time.change(min: first_fifteen)
       when 'thirty_minute'
-        first_thirty = [30, 0].detect{ |m| m <= time.min }
+        first_thirty = [30, 0].detect { |m| m <= time.min }
         time.change(min: first_thirty)
       when 'hour'
         time.beginning_of_hour

data/lib/druiddb/version.rb

@@ -0,0 +1,3 @@
+module DruidDB
+  VERSION = '1.2.0'.freeze
+end

data/lib/{druid → druiddb}/writer.rb

@@ -1,5 +1,4 @@
-#TODO: Seems to be a delay after shutting down Kafka and ZK updating
-module Druid
+module DruidDB
   class Writer
     attr_reader :config, :producer, :zk
     def initialize(config, zk)
@@ -10,28 +9,28 @@ module Druid
     end
 
     def write_point(datasource, datapoint)
-      raise Druid::ConnectionError, 'no kafka brokers available' if producer.nil?
-      producer.produce(datapoint, topic: datasource)
+      raise DruidDB::ConnectionError, 'no kafka brokers available' if producer.nil?
+      producer.produce(datapoint.to_json, topic: datasource)
     end
 
     private
 
     def broker_list
-      zk.registry["/brokers/ids"].map{|instance| "#{instance[:host]}:#{instance[:port]}" }.join(',')
+      zk.registry['/brokers/ids'].map { |instance| broker_name(instance) }.join(',')
+    end
+
+    def broker_name(instance)
+      "#{instance[:host]}:#{instance[:port]}"
     end
 
     def handle_kafka_state_change(service)
-      if service == config.kafka_broker_path
-        producer.shutdown
-        init_producer
-      end
+      return unless service == config.kafka_broker_path
+      producer.shutdown
+      init_producer
     end
 
     def init_producer
-      producer_options = {
-        seed_brokers: broker_list,
-        client_id: "ruby-druid"
-      }
+      producer_options = { seed_brokers: broker_list, client_id: config.client_id }
 
       if broker_list.present?
         kafka = Kafka.new(producer_options)

data/lib/{druid → druiddb}/zk.rb

@@ -1,9 +1,8 @@
-module Druid
+module DruidDB
   class ZK
     attr_accessor :registry
     attr_reader :client, :config, :listeners
 
-    #TODO: Test and handle ZK partitions
     def initialize(config)
       @client = ::ZK.new(config.zookeeper)
       @config = config
@@ -19,7 +18,6 @@ module Druid
     private
 
     def announce(service)
-      # puts "announcing #{service}"
       listeners.each { |listener| listener.call(service) }
     end
 
@@ -27,34 +25,28 @@ module Druid
       register_service("#{config.discovery_path}/druid:broker")
       register_service("#{config.discovery_path}/druid:coordinator")
       register_service("#{config.discovery_path}/druid:overlord")
-      register_service("#{config.kafka_broker_path}")
+      register_service(config.kafka_broker_path.to_s)
     end
 
     def register_service(service)
-      # puts "registering #{service}"
-      #TODO: Thead safety, lock this registry key
       subscribe_to_service(service)
       renew_service_instances(service)
     end
 
     def renew_service_instances(service)
-      # puts "activating registered subscriptions on #{service}"
       instances = client.children(service, watch: true)
 
-      # puts "emptying #{service} from registry"
       registry[service] = []
       instances.each do |instance|
         data = JSON.parse(client.get("#{service}/#{instance}").first)
         host = data['address'] || data['host']
         port = data['port']
-        # puts "adding #{host}:#{port} to registry for #{service}"
         registry[service] << { host: host, port: port }
       end
     end
 
     def subscribe_to_service(service)
-      subscription = client.register(service) do |event|
-        # puts "watched event for #{service} detected"
+      client.register(service) do |event|
         renew_service_instances(event.path)
         announce(event.path)
       end

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: druiddb
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Andre LeBlanc
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-07 00:00:00.000000000 Z
+date: 2017-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
         version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ">"
       - !ruby/object:Gem::Version
         version: '4.0'
 - !ruby/object:Gem::Dependency
@@ -80,8 +80,21 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-description: Ruby adapter for Druid that allows reads and writes using the Tranquility
-  Kafka API.
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+description: Ruby client for reading from and writing to Druid.
 email:
 - andre.leblanc88@gmail.com
 executables: []
@@ -89,29 +102,36 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".rubocop_todo.yml"
+- ".travis.yml"
+- Dockerfile
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
+- bin/run_tests.sh
 - bin/setup
+- docker-compose.yml
 - druiddb.gemspec
-- lib/druid/README.md
-- lib/druid/client.rb
-- lib/druid/configuration.rb
-- lib/druid/connection.rb
-- lib/druid/errors.rb
-- lib/druid/node/broker.rb
-- lib/druid/node/coordinator.rb
-- lib/druid/node/overlord.rb
-- lib/druid/queries/core.rb
-- lib/druid/queries/task.rb
-- lib/druid/query.rb
-- lib/druid/version.rb
-- lib/druid/writer.rb
-- lib/druid/zk.rb
 - lib/druiddb.rb
-homepage: https://github.com/andremleblanc/druiddb
+- lib/druiddb/client.rb
+- lib/druiddb/configuration.rb
+- lib/druiddb/connection.rb
+- lib/druiddb/errors.rb
+- lib/druiddb/node/broker.rb
+- lib/druiddb/node/coordinator.rb
+- lib/druiddb/node/overlord.rb
+- lib/druiddb/queries/core.rb
+- lib/druiddb/queries/datasources.rb
+- lib/druiddb/queries/task.rb
+- lib/druiddb/query.rb
+- lib/druiddb/version.rb
+- lib/druiddb/writer.rb
+- lib/druiddb/zk.rb
+homepage: https://github.com/andremleblanc/druiddb-ruby
 licenses:
 - MIT
 metadata: {}
@@ -134,5 +154,5 @@ rubyforge_project:
 rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
-summary: Ruby adapter for Druid.
+summary: Ruby client for Druid.
 test_files: []

data/lib/druid/README.md

@@ -1,20 +0,0 @@
-# Druid
-This module contains all logic associated with Druid.
-
-## Node
-The `Node` classes represent Druid nodes and manage connection with them. They
-also provide the methods that are exposed natively by the Druid REST API.
-
-## Query
-The query module provides a way for the `Druid::Client` to inherit the methods
-from the `Node` classes. Additionally, the `Query` module classes provide some
-additional methods not found natively in the Druid REST API.
-
-## Writer
-The `Writer` classes utilize the Tranquility Kafka API to communicate with Druid
-nodes and allows writing.
-
-## Errors
-**Client Error:** Indicates a failure within the Ruby-Druid adapter.  
-**Connection Error:** Indicates a failed request to Druid.  
-**QueryError:** Indicates a malformed query.

data/lib/druid/client.rb

@@ -1,22 +0,0 @@
-module Druid
-  class Client
-    include Druid::Queries::Core
-    include Druid::Queries::Task
-
-    attr_reader :broker,
-                :config,
-                :coordinator,
-                :overlord,
-                :writer,
-                :zk
-
-    def initialize(options = {})
-      @config = Druid::Configuration.new(options)
-      @zk = Druid::ZK.new(config)
-      @broker = Druid::Node::Broker.new(config, zk)
-      @coordinator = Druid::Node::Coordinator.new(config, zk)
-      @overlord = Druid::Node::Overlord.new(config, zk)
-      @writer = Druid::Writer.new(config, zk)
-    end
-  end
-end

data/lib/druid/queries/task.rb

@@ -1,7 +0,0 @@
-module Druid
-  module Queries
-    module Task
-      delegate :shutdown_tasks, to: :overlord
-    end
-  end
-end

data/lib/druid/version.rb

@@ -1,3 +0,0 @@
-module Druiddb
-	VERSION = '1.0.1'
-end