druid_config 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +12 -6
- data/lib/druid_config.rb +1 -0
- data/lib/druid_config/cluster.rb +66 -30
- data/lib/druid_config/entities/data_source.rb +26 -13
- data/lib/druid_config/entities/node.rb +65 -13
- data/lib/druid_config/entities/task.rb +97 -0
- data/lib/druid_config/entities/tier.rb +18 -4
- data/lib/druid_config/entities/worker.rb +12 -4
- data/lib/druid_config/util.rb +34 -0
- data/lib/druid_config/version.rb +1 -1
- data/lib/druid_config/zk.rb +6 -0
- data/spec/cluster_spec.rb +20 -24
- data/spec/data/druid_responses.yml +4 -0
- data/spec/node_spec.rb +6 -4
- data/spec/spec_helper.rb +22 -16
- metadata +5 -2
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA1:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: a03eb8b4f7df4223fe098516a23443beac52be00
         | 
| 4 | 
            +
              data.tar.gz: 230450c3d105e2327c681f1bc1beb225926361be
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: 151efb283126cbf585549a0e1d0a7f766f48c51e3f21e18ab373c34d59a247c64fbcc1188157ccaadb7b5940ccdcc480c24b87f33a2f0e890e019be817b9bcab
         | 
| 7 | 
            +
              data.tar.gz: db35a7cc044f47bb4de83ea1467e35186389471485f71f48e3f4363087dc8f9326150a0df93a7844d901b804bf38348d51ab2cfb452a2aa708240cbd08a5ba76
         | 
    
        data/README.md
    CHANGED
    
    | @@ -1,3 +1,5 @@ | |
| 1 | 
            +
            [](https://badge.fury.io/rb/druid_config) [](https://travis-ci.org/redBorder/druid_config)
         | 
| 2 | 
            +
             | 
| 1 3 | 
             
            # DruidConfig
         | 
| 2 4 |  | 
| 3 5 | 
             
            DruidConfig is a gem to access the information about Druid cluster status. You can check a node capacity, number of segments, tiers... It uses [zookeeper](https://zookeeper.apache.org/) to get coordinator and overlord URIs.
         | 
| @@ -5,9 +7,13 @@ DruidConfig is a gem to access the information about Druid cluster status. You c | |
| 5 7 | 
             
            To use in your application, add this line to your Gemfile:
         | 
| 6 8 |  | 
| 7 9 | 
             
            ```ruby
         | 
| 8 | 
            -
             | 
| 10 | 
            +
            gem 'druid_config'
         | 
| 9 11 | 
             
            ```
         | 
| 10 12 |  | 
| 13 | 
            +
            ## Query Druid data
         | 
| 14 | 
            +
             | 
| 15 | 
            +
            The purpose of this gem is to query Druid cluster status. If you want to query Druid data, I recommend you to use [ruby-druid gem](https://github.com/ruby-druid/ruby-druid).
         | 
| 16 | 
            +
             | 
| 11 17 | 
             
            # Initialization
         | 
| 12 18 |  | 
| 13 19 | 
             
            `Cluster` is the base class to perform queries. To initialize it send the Zookeeper URI and options as arguments:
         | 
| @@ -44,11 +50,11 @@ Call methods defined in `DruidConfig::Cluster` to access to the data. To get mor | |
| 44 50 |  | 
| 45 51 | 
             
            Some methods return an instance of an `Entity` class. These entities provide multiple methods to access data. Defined entities are inside `druid_config/entities` folder.
         | 
| 46 52 |  | 
| 47 | 
            -
            * [DataSource]()
         | 
| 48 | 
            -
            * [ | 
| 49 | 
            -
            * [ | 
| 50 | 
            -
            * [ | 
| 51 | 
            -
            * [Worker]()
         | 
| 53 | 
            +
            * [DataSource](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/data_source.rb)
         | 
| 54 | 
            +
            * [Segment](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/segment.rb)
         | 
| 55 | 
            +
            * [Tier](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/tier.rb)
         | 
| 56 | 
            +
            * [Node](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/node.rb)
         | 
| 57 | 
            +
            * [Worker](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/worker.rb)
         | 
| 52 58 |  | 
| 53 59 | 
             
            ## Exceptions
         | 
| 54 60 |  | 
    
        data/lib/druid_config.rb
    CHANGED
    
    | @@ -6,6 +6,7 @@ require 'druid_config/zk' | |
| 6 6 | 
             
            require 'druid_config/version'
         | 
| 7 7 | 
             
            require 'druid_config/util'
         | 
| 8 8 | 
             
            require 'druid_config/entities/segment'
         | 
| 9 | 
            +
            require 'druid_config/entities/task'
         | 
| 9 10 | 
             
            require 'druid_config/entities/worker'
         | 
| 10 11 | 
             
            require 'druid_config/entities/node'
         | 
| 11 12 | 
             
            require 'druid_config/entities/tier'
         | 
    
        data/lib/druid_config/cluster.rb
    CHANGED
    
    | @@ -136,7 +136,7 @@ module DruidConfig | |
| 136 136 | 
             
                def datasources
         | 
| 137 137 | 
             
                  datasource_status = load_status
         | 
| 138 138 | 
             
                  secure_query do
         | 
| 139 | 
            -
                    self.class.get('/datasources? | 
| 139 | 
            +
                    self.class.get('/datasources?simple').map do |data|
         | 
| 140 140 | 
             
                      DruidConfig::Entities::DataSource.new(
         | 
| 141 141 | 
             
                        data,
         | 
| 142 142 | 
             
                        datasource_status.select { |k, _| k == data['name'] }.values.first)
         | 
| @@ -202,8 +202,8 @@ module DruidConfig | |
| 202 202 | 
             
                #
         | 
| 203 203 | 
             
                def servers
         | 
| 204 204 | 
             
                  secure_query do
         | 
| 205 | 
            -
                    queue = load_queue(' | 
| 206 | 
            -
                    self.class.get('/servers? | 
| 205 | 
            +
                    queue = load_queue('simple')
         | 
| 206 | 
            +
                    self.class.get('/servers?simple').map do |data|
         | 
| 207 207 | 
             
                      DruidConfig::Entities::Node.new(
         | 
| 208 208 | 
             
                        data,
         | 
| 209 209 | 
             
                        queue.select { |k, _| k == data['host'] }.values.first)
         | 
| @@ -253,22 +253,14 @@ module DruidConfig | |
| 253 253 | 
             
                # Array of Workers
         | 
| 254 254 | 
             
                #
         | 
| 255 255 | 
             
                def workers
         | 
| 256 | 
            -
                  # Stash the base_uri
         | 
| 257 | 
            -
                  stash_uri
         | 
| 258 | 
            -
                  self.class.base_uri(
         | 
| 259 | 
            -
                    "#{DruidConfig.client.overlord}"\
         | 
| 260 | 
            -
                    "druid/indexer/#{DruidConfig::Version::API_VERSION}")
         | 
| 261 256 | 
             
                  workers = []
         | 
| 262 257 | 
             
                  # Perform a query
         | 
| 263 | 
            -
                   | 
| 258 | 
            +
                  query_overlord do
         | 
| 264 259 | 
             
                    secure_query do
         | 
| 265 260 | 
             
                      workers = self.class.get('/workers').map do |worker|
         | 
| 266 261 | 
             
                        DruidConfig::Entities::Worker.new(worker)
         | 
| 267 262 | 
             
                      end
         | 
| 268 263 | 
             
                    end
         | 
| 269 | 
            -
                  ensure
         | 
| 270 | 
            -
                    # Recover it
         | 
| 271 | 
            -
                    pop_uri
         | 
| 272 264 | 
             
                  end
         | 
| 273 265 | 
             
                  # Return
         | 
| 274 266 | 
             
                  workers
         | 
| @@ -281,6 +273,68 @@ module DruidConfig | |
| 281 273 | 
             
                  @physical_workers ||= workers.map(&:host).uniq
         | 
| 282 274 | 
             
                end
         | 
| 283 275 |  | 
| 276 | 
            +
                # Tasks
         | 
| 277 | 
            +
                # -----------------
         | 
| 278 | 
            +
                
         | 
| 279 | 
            +
                #
         | 
| 280 | 
            +
                # Return tasks based on the status
         | 
| 281 | 
            +
                #
         | 
| 282 | 
            +
                # == Returns:
         | 
| 283 | 
            +
                # Array of Tasks
         | 
| 284 | 
            +
                #
         | 
| 285 | 
            +
                %w(running pending waiting).each do |status|
         | 
| 286 | 
            +
                  define_method("#{status}_tasks") do
         | 
| 287 | 
            +
                    tasks = []
         | 
| 288 | 
            +
                    query_overlord do
         | 
| 289 | 
            +
                      tasks = self.class.get("/#{status}Tasks").map do |task|
         | 
| 290 | 
            +
                        DruidConfig::Entities::Task.new(
         | 
| 291 | 
            +
                          task['id'],
         | 
| 292 | 
            +
                          DruidConfig::Entities::Task::STATUS[status.to_sym],
         | 
| 293 | 
            +
                          created_time: task['createdTime'],
         | 
| 294 | 
            +
                          query_insertion_time: task['queueInsertionTime'])
         | 
| 295 | 
            +
                      end
         | 
| 296 | 
            +
                    end
         | 
| 297 | 
            +
                  end
         | 
| 298 | 
            +
                end
         | 
| 299 | 
            +
             | 
| 300 | 
            +
                #
         | 
| 301 | 
            +
                # Find a task
         | 
| 302 | 
            +
                #
         | 
| 303 | 
            +
                def task(id)
         | 
| 304 | 
            +
                  DruidConfig::Entities::Task.find(id)
         | 
| 305 | 
            +
                end
         | 
| 306 | 
            +
                
         | 
| 307 | 
            +
                #
         | 
| 308 | 
            +
                # Return complete tasks
         | 
| 309 | 
            +
                #
         | 
| 310 | 
            +
                # == Returns:
         | 
| 311 | 
            +
                # Array of Tasks
         | 
| 312 | 
            +
                #
         | 
| 313 | 
            +
                def complete_tasks
         | 
| 314 | 
            +
                  tasks = []
         | 
| 315 | 
            +
                  query_overlord do
         | 
| 316 | 
            +
                    tasks = self.class.get('/completeTasks').map do |task|
         | 
| 317 | 
            +
                      DruidConfig::Entities::Task.new(
         | 
| 318 | 
            +
                        task['id'],
         | 
| 319 | 
            +
                        task['statusCode'])
         | 
| 320 | 
            +
                    end
         | 
| 321 | 
            +
                  end
         | 
| 322 | 
            +
                end
         | 
| 323 | 
            +
             | 
| 324 | 
            +
                #
         | 
| 325 | 
            +
                # Return failed completed tasks
         | 
| 326 | 
            +
                #
         | 
| 327 | 
            +
                def failed_tasks
         | 
| 328 | 
            +
                  complete_tasks.select(&:failed?)
         | 
| 329 | 
            +
                end
         | 
| 330 | 
            +
             | 
| 331 | 
            +
                #
         | 
| 332 | 
            +
                # Return success completed tasks
         | 
| 333 | 
            +
                #
         | 
| 334 | 
            +
                def success_tasks
         | 
| 335 | 
            +
                  complete_tasks.select(&:success?)
         | 
| 336 | 
            +
                end
         | 
| 337 | 
            +
             | 
| 284 338 | 
             
                # Services
         | 
| 285 339 | 
             
                # -----------------
         | 
| 286 340 |  | 
| @@ -302,23 +356,5 @@ module DruidConfig | |
| 302 356 | 
             
                  # Return nodes
         | 
| 303 357 | 
             
                  @services = services
         | 
| 304 358 | 
             
                end
         | 
| 305 | 
            -
             | 
| 306 | 
            -
                private
         | 
| 307 | 
            -
             | 
| 308 | 
            -
                #
         | 
| 309 | 
            -
                # Stash current base_uri
         | 
| 310 | 
            -
                #
         | 
| 311 | 
            -
                def stash_uri
         | 
| 312 | 
            -
                  @uri_stack ||= []
         | 
| 313 | 
            -
                  @uri_stack.push self.class.base_uri
         | 
| 314 | 
            -
                end
         | 
| 315 | 
            -
             | 
| 316 | 
            -
                #
         | 
| 317 | 
            -
                # Pop next base_uri
         | 
| 318 | 
            -
                #
         | 
| 319 | 
            -
                def pop_uri
         | 
| 320 | 
            -
                  return if @uri_stack.nil? || @uri_stack.empty?
         | 
| 321 | 
            -
                  self.class.base_uri(@uri_stack.pop)
         | 
| 322 | 
            -
                end
         | 
| 323 359 | 
             
              end
         | 
| 324 360 | 
             
            end
         | 
| @@ -1,6 +1,6 @@ | |
| 1 1 | 
             
            module DruidConfig
         | 
| 2 2 | 
             
              #
         | 
| 3 | 
            -
              # Module of  | 
| 3 | 
            +
              # Module of Entities
         | 
| 4 4 | 
             
              #
         | 
| 5 5 | 
             
              module Entities
         | 
| 6 6 | 
             
                #
         | 
| @@ -8,6 +8,7 @@ module DruidConfig | |
| 8 8 | 
             
                class DataSource
         | 
| 9 9 | 
             
                  # HTTParty Rocks!
         | 
| 10 10 | 
             
                  include HTTParty
         | 
| 11 | 
            +
                  include DruidConfig::Util
         | 
| 11 12 |  | 
| 12 13 | 
             
                  attr_reader :name, :properties, :load_status
         | 
| 13 14 |  | 
| @@ -31,28 +32,36 @@ module DruidConfig | |
| 31 32 | 
             
                  # http://druid.io/docs/0.8.1/design/coordinator.html
         | 
| 32 33 | 
             
                  #
         | 
| 33 34 |  | 
| 34 | 
            -
                  def info | 
| 35 | 
            -
                     | 
| 35 | 
            +
                  def info
         | 
| 36 | 
            +
                    secure_query do
         | 
| 37 | 
            +
                      @info ||= self.class.get("/datasources/#{@name}")
         | 
| 38 | 
            +
                    end
         | 
| 36 39 | 
             
                  end
         | 
| 37 40 |  | 
| 38 41 | 
             
                  # Intervals
         | 
| 39 42 | 
             
                  # -----------------
         | 
| 40 43 | 
             
                  def intervals(params = '')
         | 
| 41 | 
            -
                     | 
| 44 | 
            +
                    secure_query do
         | 
| 45 | 
            +
                      self.class.get("/datasources/#{@name}/intervals?#{params}")
         | 
| 46 | 
            +
                    end
         | 
| 42 47 | 
             
                  end
         | 
| 43 48 |  | 
| 44 49 | 
             
                  def interval(interval, params = '')
         | 
| 45 | 
            -
                     | 
| 46 | 
            -
             | 
| 50 | 
            +
                    secure_query do
         | 
| 51 | 
            +
                      self.class.get("/datasources/#{@name}/intervals/#{interval}"\
         | 
| 52 | 
            +
                                     "?#{params}")
         | 
| 53 | 
            +
                    end
         | 
| 47 54 | 
             
                  end
         | 
| 48 55 |  | 
| 49 56 | 
             
                  # Segments and Tiers
         | 
| 50 57 | 
             
                  # -----------------
         | 
| 51 58 | 
             
                  def segments
         | 
| 52 | 
            -
                     | 
| 53 | 
            -
                       | 
| 54 | 
            -
                         | 
| 55 | 
            -
             | 
| 59 | 
            +
                    secure_query do
         | 
| 60 | 
            +
                      @segments ||=
         | 
| 61 | 
            +
                        self.class.get("/datasources/#{@name}/segments?full").map do |s|
         | 
| 62 | 
            +
                          DruidConfig::Entities::Segment.new(s)
         | 
| 63 | 
            +
                        end
         | 
| 64 | 
            +
                    end
         | 
| 56 65 | 
             
                  end
         | 
| 57 66 |  | 
| 58 67 | 
             
                  def segment(segment)
         | 
| @@ -66,12 +75,16 @@ module DruidConfig | |
| 66 75 | 
             
                  # Rules
         | 
| 67 76 | 
             
                  # -----------------
         | 
| 68 77 | 
             
                  def rules(params = '')
         | 
| 69 | 
            -
                     | 
| 78 | 
            +
                    secure_query do
         | 
| 79 | 
            +
                      self.class.get("/rules/#{@name}?#{params}")
         | 
| 80 | 
            +
                    end
         | 
| 70 81 | 
             
                  end
         | 
| 71 82 |  | 
| 72 83 | 
             
                  def history_rules(interval)
         | 
| 73 | 
            -
                     | 
| 74 | 
            -
             | 
| 84 | 
            +
                    secure_query do
         | 
| 85 | 
            +
                      self.class.get("/rules/#{@name}/history"\
         | 
| 86 | 
            +
                                     "?interval=#{interval}")
         | 
| 87 | 
            +
                    end
         | 
| 75 88 | 
             
                  end
         | 
| 76 89 | 
             
                end
         | 
| 77 90 | 
             
              end
         | 
| @@ -6,10 +6,11 @@ module DruidConfig | |
| 6 6 | 
             
                class Node
         | 
| 7 7 | 
             
                  # HTTParty Rocks!
         | 
| 8 8 | 
             
                  include HTTParty
         | 
| 9 | 
            +
                  include DruidConfig::Util
         | 
| 9 10 |  | 
| 10 11 | 
             
                  # Readers
         | 
| 11 12 | 
             
                  attr_reader :host, :port, :max_size, :type, :tier, :priority, :size,
         | 
| 12 | 
            -
                              : | 
| 13 | 
            +
                              :segments_to_load_count, :segments_to_drop_count,
         | 
| 13 14 | 
             
                              :segments_to_load_size, :segments_to_drop_size
         | 
| 14 15 |  | 
| 15 16 | 
             
                  #
         | 
| @@ -28,21 +29,20 @@ module DruidConfig | |
| 28 29 | 
             
                    @tier = metadata['tier']
         | 
| 29 30 | 
             
                    @priority = metadata['priority']
         | 
| 30 31 | 
             
                    @size = metadata['currSize']
         | 
| 31 | 
            -
                     | 
| 32 | 
            -
             | 
| 33 | 
            -
             | 
| 32 | 
            +
                    # Set end point for HTTParty
         | 
| 33 | 
            +
                    self.class.base_uri(
         | 
| 34 | 
            +
                      "#{DruidConfig.client.coordinator}"\
         | 
| 35 | 
            +
                      "druid/coordinator/#{DruidConfig::Version::API_VERSION}")
         | 
| 36 | 
            +
             | 
| 37 | 
            +
                    # Load more data from queue
         | 
| 34 38 | 
             
                    if queue.nil?
         | 
| 35 | 
            -
                      @ | 
| 39 | 
            +
                      @segments_to_load_count, @segments_to_drop_count = 0, 0
         | 
| 36 40 | 
             
                      @segments_to_load_size, @segments_to_drop_size = 0, 0
         | 
| 37 41 | 
             
                    else
         | 
| 38 | 
            -
                      @ | 
| 39 | 
            -
             | 
| 40 | 
            -
                       | 
| 41 | 
            -
                      @ | 
| 42 | 
            -
                        DruidConfig::Entities::Segment.new(segment)
         | 
| 43 | 
            -
                      end
         | 
| 44 | 
            -
                      @segments_to_load_size = @segments_to_load.map(&:size).reduce(:+)
         | 
| 45 | 
            -
                      @segments_to_drop_size = @segments_to_drop.map(&:size).reduce(:+)
         | 
| 42 | 
            +
                      @segments_to_load_count = queue['segmentsToLoad']
         | 
| 43 | 
            +
                      @segments_to_drop_count = queue['segmentsToDrop']
         | 
| 44 | 
            +
                      @segments_to_load_size = queue['segmentsToLoadSize']
         | 
| 45 | 
            +
                      @segments_to_drop_size = queue['segmentsToLoadSize']
         | 
| 46 46 | 
             
                    end
         | 
| 47 47 | 
             
                  end
         | 
| 48 48 |  | 
| @@ -63,12 +63,64 @@ module DruidConfig | |
| 63 63 | 
             
                    max_size - size
         | 
| 64 64 | 
             
                  end
         | 
| 65 65 |  | 
| 66 | 
            +
                  #
         | 
| 67 | 
            +
                  # Return all segments of this node
         | 
| 68 | 
            +
                  #
         | 
| 69 | 
            +
                  def segments_count
         | 
| 70 | 
            +
                    @segments_count ||=
         | 
| 71 | 
            +
                      self.class.get("/servers/#{uri}/segments").size
         | 
| 72 | 
            +
                  end
         | 
| 73 | 
            +
             | 
| 74 | 
            +
                  #
         | 
| 75 | 
            +
                  # Return all segments of this node
         | 
| 76 | 
            +
                  #
         | 
| 77 | 
            +
                  def segments
         | 
| 78 | 
            +
                    @segments ||=
         | 
| 79 | 
            +
                      self.class.get("/servers/#{uri}/segments?full").map do |s|
         | 
| 80 | 
            +
                        DruidConfig::Entities::Segment.new(s)
         | 
| 81 | 
            +
                      end
         | 
| 82 | 
            +
                  end
         | 
| 83 | 
            +
             | 
| 84 | 
            +
                  #
         | 
| 85 | 
            +
                  # Get segments to load
         | 
| 86 | 
            +
                  #
         | 
| 87 | 
            +
                  def segments_to_load
         | 
| 88 | 
            +
                    current_queue = queue
         | 
| 89 | 
            +
                    return [] unless current_queue
         | 
| 90 | 
            +
                    current_queue['segmentsToLoad'].map do |segment|
         | 
| 91 | 
            +
                      DruidConfig::Entities::Segment.new(segment)
         | 
| 92 | 
            +
                    end
         | 
| 93 | 
            +
                  end
         | 
| 94 | 
            +
             | 
| 95 | 
            +
                  #
         | 
| 96 | 
            +
                  # Get segments to drop
         | 
| 97 | 
            +
                  #
         | 
| 98 | 
            +
                  def segments_to_drop
         | 
| 99 | 
            +
                    current_queue = queue
         | 
| 100 | 
            +
                    return [] unless current_queue
         | 
| 101 | 
            +
                    current_queue['segmentsToDrop'].map do |segment|
         | 
| 102 | 
            +
                      DruidConfig::Entities::Segment.new(segment)
         | 
| 103 | 
            +
                    end
         | 
| 104 | 
            +
                  end
         | 
| 105 | 
            +
             | 
| 66 106 | 
             
                  #
         | 
| 67 107 | 
             
                  # Return the URI of this node
         | 
| 68 108 | 
             
                  #
         | 
| 69 109 | 
             
                  def uri
         | 
| 70 110 | 
             
                    "#{@host}:#{@port}"
         | 
| 71 111 | 
             
                  end
         | 
| 112 | 
            +
             | 
| 113 | 
            +
                  private
         | 
| 114 | 
            +
             | 
| 115 | 
            +
                  #
         | 
| 116 | 
            +
                  # Get load queue for this node
         | 
| 117 | 
            +
                  #
         | 
| 118 | 
            +
                  def queue
         | 
| 119 | 
            +
                    secure_query do
         | 
| 120 | 
            +
                      self.class.get('/loadqueue?full')
         | 
| 121 | 
            +
                        .select { |k, _| k == uri }.values.first
         | 
| 122 | 
            +
                    end
         | 
| 123 | 
            +
                  end
         | 
| 72 124 | 
             
                end
         | 
| 73 125 | 
             
              end
         | 
| 74 126 | 
             
            end
         | 
| @@ -0,0 +1,97 @@ | |
| 1 | 
            +
            module DruidConfig
         | 
| 2 | 
            +
              #
         | 
| 3 | 
            +
              # Module of Entities
         | 
| 4 | 
            +
              #
         | 
| 5 | 
            +
              module Entities
         | 
| 6 | 
            +
                #
         | 
| 7 | 
            +
                # Init a DataSource
         | 
| 8 | 
            +
                class Task
         | 
| 9 | 
            +
                  # HTTParty Rocks!
         | 
| 10 | 
            +
                  include HTTParty
         | 
| 11 | 
            +
                  include DruidConfig::Util
         | 
| 12 | 
            +
             | 
| 13 | 
            +
                  # Statuses constants
         | 
| 14 | 
            +
                  STATUS = {
         | 
| 15 | 
            +
                    running: 'RUNNING',
         | 
| 16 | 
            +
                    pending: 'PENDING',
         | 
| 17 | 
            +
                    success: 'SUCCESS',
         | 
| 18 | 
            +
                    waiting: 'WAITING',
         | 
| 19 | 
            +
                    failed: 'FAILED'
         | 
| 20 | 
            +
                  }
         | 
| 21 | 
            +
             | 
| 22 | 
            +
                  attr_reader :id, :status, :created_time, :query_insertion_time
         | 
| 23 | 
            +
             | 
| 24 | 
            +
                  #
         | 
| 25 | 
            +
                  # Find a task based on id
         | 
| 26 | 
            +
                  #
         | 
| 27 | 
            +
                  def self.find(id)
         | 
| 28 | 
            +
                    # Set end point for HTTParty
         | 
| 29 | 
            +
                    base_uri(
         | 
| 30 | 
            +
                      "#{DruidConfig.client.overlord}"\
         | 
| 31 | 
            +
                      "druid/indexer/#{DruidConfig::Version::API_VERSION}/task")
         | 
| 32 | 
            +
                    # Get data
         | 
| 33 | 
            +
                    status = get("/#{id}/status")
         | 
| 34 | 
            +
                    new(id, status['status']['status'])
         | 
| 35 | 
            +
                  end
         | 
| 36 | 
            +
             | 
| 37 | 
            +
                  #
         | 
| 38 | 
            +
                  # Initialize a task
         | 
| 39 | 
            +
                  #
         | 
| 40 | 
            +
                  # == Parameters:
         | 
| 41 | 
            +
                  # id::
         | 
| 42 | 
            +
                  #   String with identifier
         | 
| 43 | 
            +
                  # status::
         | 
| 44 | 
            +
                  #   Current status of the task
         | 
| 45 | 
            +
                  #
         | 
| 46 | 
            +
                  def initialize(id, status, extended_info = {})
         | 
| 47 | 
            +
                    @id = id
         | 
| 48 | 
            +
                    @status = status.downcase.to_sym
         | 
| 49 | 
            +
                    @created_time = extended_info[:created_time]
         | 
| 50 | 
            +
                    @query_insertion_time = extended_info[:query_insertion_time]
         | 
| 51 | 
            +
                    # Set end point for HTTParty
         | 
| 52 | 
            +
                    self.class.base_uri(
         | 
| 53 | 
            +
                      "#{DruidConfig.client.overlord}"\
         | 
| 54 | 
            +
                      "druid/indexer/#{DruidConfig::Version::API_VERSION}/task")
         | 
| 55 | 
            +
                  end
         | 
| 56 | 
            +
             | 
| 57 | 
            +
                  #
         | 
| 58 | 
            +
                  # Multiple methods to check status of the tasks
         | 
| 59 | 
            +
                  #
         | 
| 60 | 
            +
                  STATUS.keys.each do |s|
         | 
| 61 | 
            +
                    define_method("#{s}?") do
         | 
| 62 | 
            +
                      s.to_sym == @status
         | 
| 63 | 
            +
                    end
         | 
| 64 | 
            +
                  end
         | 
| 65 | 
            +
             | 
| 66 | 
            +
                  alias_method :completed?, :success?
         | 
| 67 | 
            +
             | 
| 68 | 
            +
                  #
         | 
| 69 | 
            +
                  # Get payload of the task
         | 
| 70 | 
            +
                  #
         | 
| 71 | 
            +
                  def payload
         | 
| 72 | 
            +
                    @payload ||= self.class.get("/#{@id}")['payload']
         | 
| 73 | 
            +
                  end
         | 
| 74 | 
            +
             | 
| 75 | 
            +
                  #
         | 
| 76 | 
            +
                  # Current log of a task
         | 
| 77 | 
            +
                  #
         | 
| 78 | 
            +
                  def log
         | 
| 79 | 
            +
                    self.class.get("/#{@id}/log")
         | 
| 80 | 
            +
                  end
         | 
| 81 | 
            +
             | 
| 82 | 
            +
                  #
         | 
| 83 | 
            +
                  # Get the dataSource of this task
         | 
| 84 | 
            +
                  #
         | 
| 85 | 
            +
                  def datasource
         | 
| 86 | 
            +
                    payload['dataSource']
         | 
| 87 | 
            +
                  end
         | 
| 88 | 
            +
             | 
| 89 | 
            +
                  #
         | 
| 90 | 
            +
                  # Group of the task
         | 
| 91 | 
            +
                  #
         | 
| 92 | 
            +
                  def group_id
         | 
| 93 | 
            +
                    payload['groupId']
         | 
| 94 | 
            +
                  end
         | 
| 95 | 
            +
                end
         | 
| 96 | 
            +
              end
         | 
| 97 | 
            +
            end
         | 
| @@ -42,14 +42,28 @@ module DruidConfig | |
| 42 42 | 
             
                                  .flatten.sort_by { |seg| seg.interval.first }
         | 
| 43 43 | 
             
                  end
         | 
| 44 44 |  | 
| 45 | 
            +
                  def segments_count
         | 
| 46 | 
            +
                    @segments_count ||= nodes.map(&:segments_count).inject(:+)
         | 
| 47 | 
            +
                  end
         | 
| 48 | 
            +
             | 
| 45 49 | 
             
                  def segments_to_load
         | 
| 46 | 
            -
                    @segments_to_load ||=
         | 
| 47 | 
            -
             | 
| 50 | 
            +
                    @segments_to_load ||= nodes.map(&:segments_to_load)
         | 
| 51 | 
            +
                                          .flatten.sort_by { |seg| seg.interval.first }
         | 
| 48 52 | 
             
                  end
         | 
| 49 53 |  | 
| 50 54 | 
             
                  def segments_to_drop
         | 
| 51 | 
            -
                    @segments_to_drop ||=
         | 
| 52 | 
            -
             | 
| 55 | 
            +
                    @segments_to_drop ||= nodes.map(&:segments_to_drop)
         | 
| 56 | 
            +
                                          .flatten.sort_by { |seg| seg.interval.first }
         | 
| 57 | 
            +
                  end
         | 
| 58 | 
            +
             | 
| 59 | 
            +
                  def segments_to_load_count
         | 
| 60 | 
            +
                    @segments_to_load_count ||=
         | 
| 61 | 
            +
                      nodes.map(&:segments_to_load_count).inject(:+)
         | 
| 62 | 
            +
                  end
         | 
| 63 | 
            +
             | 
| 64 | 
            +
                  def segments_to_drop_count
         | 
| 65 | 
            +
                    @segments_to_drop_count ||=
         | 
| 66 | 
            +
                      nodes.map(&:segments_to_drop_count).inject(:+)
         | 
| 53 67 | 
             
                  end
         | 
| 54 68 |  | 
| 55 69 | 
             
                  def segments_to_load_size
         | 
| @@ -6,7 +6,7 @@ module DruidConfig | |
| 6 6 | 
             
                class Worker
         | 
| 7 7 | 
             
                  # Readers
         | 
| 8 8 | 
             
                  attr_reader :last_completed_task_time, :host, :port, :ip, :capacity,
         | 
| 9 | 
            -
             | 
| 9 | 
            +
                              :version, :running_tasks, :capacity_used
         | 
| 10 10 |  | 
| 11 11 | 
             
                  #
         | 
| 12 12 | 
             
                  # Initialize it with received info
         | 
| @@ -21,21 +21,29 @@ module DruidConfig | |
| 21 21 | 
             
                    @capacity = metadata['worker']['capacity']
         | 
| 22 22 | 
             
                    @version = metadata['worker']['version']
         | 
| 23 23 | 
             
                    @last_completed_task_time = metadata['lastCompletedTaskTime']
         | 
| 24 | 
            -
                    @running_tasks = metadata['runningTasks']
         | 
| 24 | 
            +
                    @running_tasks = metadata['runningTasks'].map do |task|
         | 
| 25 | 
            +
                      DruidConfig::Entities::Task.new(
         | 
| 26 | 
            +
                        task,
         | 
| 27 | 
            +
                        DruidConfig::Entities::Task::STATUS[:running],
         | 
| 28 | 
            +
                        created_time: task['createdTime'],
         | 
| 29 | 
            +
                        query_insertion_time: task['queueInsertionTime'])
         | 
| 30 | 
            +
                    end
         | 
| 25 31 | 
             
                    @capacity_used = metadata['currCapacityUsed']
         | 
| 26 32 | 
             
                  end
         | 
| 27 33 |  | 
| 34 | 
            +
                  alias_method :used, :capacity_used
         | 
| 35 | 
            +
             | 
| 28 36 | 
             
                  #
         | 
| 29 37 | 
             
                  # Return free capacity
         | 
| 30 38 | 
             
                  #
         | 
| 31 39 | 
             
                  def free
         | 
| 32 | 
            -
                    @free ||= (capacity -  | 
| 40 | 
            +
                    @free ||= (capacity - capacity_used)
         | 
| 33 41 | 
             
                  end
         | 
| 34 42 |  | 
| 35 43 | 
             
                  #
         | 
| 36 44 | 
             
                  # Return capacity used
         | 
| 37 45 | 
             
                  #
         | 
| 38 | 
            -
                  def  | 
| 46 | 
            +
                  def used_percent
         | 
| 39 47 | 
             
                    return 0 unless @capacity && @capacity != 0
         | 
| 40 48 | 
             
                    ((@capacity_used.to_f / @capacity) * 100).round(2)
         | 
| 41 49 | 
             
                  end
         | 
    
        data/lib/druid_config/util.rb
    CHANGED
    
    | @@ -31,5 +31,39 @@ module DruidConfig | |
| 31 31 | 
             
                    retry
         | 
| 32 32 | 
             
                  end
         | 
| 33 33 | 
             
                end
         | 
| 34 | 
            +
             | 
| 35 | 
            +
                #
         | 
| 36 | 
            +
                # Update the URI of HTTParty to perform queries to Overlord.
         | 
| 37 | 
            +
                # After perform the query, the URI is reverted to coordinator.
         | 
| 38 | 
            +
                #
         | 
| 39 | 
            +
                def query_overlord
         | 
| 40 | 
            +
                  return unless block_given?
         | 
| 41 | 
            +
                  stash_uri
         | 
| 42 | 
            +
                  self.class.base_uri(
         | 
| 43 | 
            +
                    "#{DruidConfig.client.overlord}"\
         | 
| 44 | 
            +
                    "druid/indexer/#{DruidConfig::Version::API_VERSION}")
         | 
| 45 | 
            +
                  begin
         | 
| 46 | 
            +
                    yield
         | 
| 47 | 
            +
                  ensure
         | 
| 48 | 
            +
                    # Ensure we revert the URI
         | 
| 49 | 
            +
                    pop_uri
         | 
| 50 | 
            +
                  end
         | 
| 51 | 
            +
                end
         | 
| 52 | 
            +
             | 
| 53 | 
            +
                #
         | 
| 54 | 
            +
                # Stash current base_uri
         | 
| 55 | 
            +
                #
         | 
| 56 | 
            +
                def stash_uri
         | 
| 57 | 
            +
                  @uri_stack ||= []
         | 
| 58 | 
            +
                  @uri_stack.push self.class.base_uri
         | 
| 59 | 
            +
                end
         | 
| 60 | 
            +
             | 
| 61 | 
            +
                #
         | 
| 62 | 
            +
                # Pop next base_uri
         | 
| 63 | 
            +
                #
         | 
| 64 | 
            +
                def pop_uri
         | 
| 65 | 
            +
                  return if @uri_stack.nil? || @uri_stack.empty?
         | 
| 66 | 
            +
                  self.class.base_uri(@uri_stack.pop)
         | 
| 67 | 
            +
                end
         | 
| 34 68 | 
             
              end
         | 
| 35 69 | 
             
            end
         | 
    
        data/lib/druid_config/version.rb
    CHANGED
    
    
    
        data/lib/druid_config/zk.rb
    CHANGED
    
    
    
        data/spec/cluster_spec.rb
    CHANGED
    
    | @@ -2,31 +2,27 @@ require 'spec_helper' | |
| 2 2 | 
             
            require 'pry'
         | 
| 3 3 | 
             
            require 'pry-nav'
         | 
| 4 4 |  | 
| 5 | 
            -
             | 
| 6 | 
            -
             | 
| 7 | 
            -
             | 
| 8 | 
            -
             | 
| 5 | 
            +
            describe DruidConfig::Cluster do
         | 
| 6 | 
            +
              before(:each) do
         | 
| 7 | 
            +
                @cluster = DruidConfig::Cluster.new('localhost', zk_keepalive: true)
         | 
| 8 | 
            +
              end
         | 
| 9 9 |  | 
| 10 | 
            -
             | 
| 11 | 
            -
             | 
| 12 | 
            -
             | 
| 10 | 
            +
              it 'must get the leader' do
         | 
| 11 | 
            +
                expect(@cluster.leader).to eq 'coordinator.stub'
         | 
| 12 | 
            +
              end
         | 
| 13 13 |  | 
| 14 | 
            -
             | 
| 15 | 
            -
             | 
| 14 | 
            +
              it 'must get load status' do
         | 
| 15 | 
            +
                basic = @cluster.load_status
         | 
| 16 | 
            +
                expect(basic.keys).to eq %w(datasource1 datasource2)
         | 
| 17 | 
            +
                expect(basic[basic.keys.first]).to eq 100
         | 
| 16 18 |  | 
| 19 | 
            +
                simple = @cluster.load_status('simple')
         | 
| 20 | 
            +
                expect(simple.keys).to eq %w(datasource1 datasource2)
         | 
| 21 | 
            +
                expect(simple[simple.keys.first]).to eq 0
         | 
| 17 22 |  | 
| 18 | 
            -
             | 
| 19 | 
            -
             | 
| 20 | 
            -
             | 
| 21 | 
            -
             | 
| 22 | 
            -
             | 
| 23 | 
            -
             | 
| 24 | 
            -
            #     # expect(simple[simple.keys.first]).to eq 0
         | 
| 25 | 
            -
             | 
| 26 | 
            -
            #     # # Use tiers
         | 
| 27 | 
            -
            #     # simple = @cluster.load_status('full')
         | 
| 28 | 
            -
            #     # expect(simple.keys).to eq %w(_default_tier hot)
         | 
| 29 | 
            -
            #     # expect(simple['_default_tier']['datasource1']).to eq 0
         | 
| 30 | 
            -
            #     # expect(simple['hot']['datasource2']).to eq 0
         | 
| 31 | 
            -
            #   end
         | 
| 32 | 
            -
            # end
         | 
| 23 | 
            +
                full = @cluster.load_status('full')
         | 
| 24 | 
            +
                expect(full.keys).to eq %w(_default_tier hot)
         | 
| 25 | 
            +
                expect(full['_default_tier']['datasource1']).to eq 0
         | 
| 26 | 
            +
                expect(full['hot']['datasource2']).to eq 0
         | 
| 27 | 
            +
              end
         | 
| 28 | 
            +
            end
         | 
    
        data/spec/node_spec.rb
    CHANGED
    
    | @@ -28,8 +28,10 @@ describe DruidConfig::Entities::Node do | |
| 28 28 | 
             
                              'priority' => 0, 'segments' => @segments, 'currSize' => @size }
         | 
| 29 29 |  | 
| 30 30 | 
             
                @queue = {
         | 
| 31 | 
            -
                  'segmentsToLoad' =>  | 
| 32 | 
            -
                  'segmentsToDrop' =>  | 
| 31 | 
            +
                  'segmentsToLoad' => 0,
         | 
| 32 | 
            +
                  'segmentsToDrop' => 0,
         | 
| 33 | 
            +
                  'segmentsToLoadSize' => 0,
         | 
| 34 | 
            +
                  'segmentsToDropSize' => 0
         | 
| 33 35 | 
             
                }
         | 
| 34 36 | 
             
              end
         | 
| 35 37 |  | 
| @@ -41,8 +43,8 @@ describe DruidConfig::Entities::Node do | |
| 41 43 | 
             
                expect(datasource.type).to eq @type.to_sym
         | 
| 42 44 | 
             
                expect(datasource.priority).to eq @priority
         | 
| 43 45 | 
             
                expect(datasource.size).to eq @size
         | 
| 44 | 
            -
                expect(datasource. | 
| 45 | 
            -
                expect(datasource. | 
| 46 | 
            +
                expect(datasource.segments_to_load_count).to eq 0
         | 
| 47 | 
            +
                expect(datasource.segments_to_drop_count).to eq 0
         | 
| 46 48 | 
             
              end
         | 
| 47 49 |  | 
| 48 50 | 
             
              it 'calculate free space' do
         | 
    
        data/spec/spec_helper.rb
    CHANGED
    
    | @@ -46,8 +46,11 @@ RSpec.configure do |config| | |
| 46 46 | 
             
                    def coordinator
         | 
| 47 47 | 
             
                      'coordinator.stub/'
         | 
| 48 48 | 
             
                    end
         | 
| 49 | 
            +
                    def overlord
         | 
| 50 | 
            +
                      'overlord.stub/'
         | 
| 51 | 
            +
                    end
         | 
| 49 52 | 
             
                  end
         | 
| 50 | 
            -
                  allow(DruidConfig).to receive(: | 
| 53 | 
            +
                  allow(DruidConfig::Client).to receive(:new) { ClientStub.new }
         | 
| 51 54 |  | 
| 52 55 | 
             
                  # Stub queries
         | 
| 53 56 | 
             
                  # ----------------------------------
         | 
| @@ -56,24 +59,27 @@ RSpec.configure do |config| | |
| 56 59 | 
             
                  # leader: coordinator.stub
         | 
| 57 60 | 
             
                  # datasources: datasource1, datasource2
         | 
| 58 61 | 
             
                  # tiers: _default_tier, hot
         | 
| 59 | 
            -
                   | 
| 60 | 
            -
                  # | 
| 61 | 
            -
                   | 
| 62 | 
            +
                  
         | 
| 63 | 
            +
                  # Load data
         | 
| 64 | 
            +
                  responses = YAML.load(File.read('spec/data/druid_responses.yml'))
         | 
| 65 | 
            +
                  head = { 'Accept' => '*/*', 'User-Agent' => 'Ruby' }
         | 
| 66 | 
            +
                  rhead = { 'Content-Type' => 'application/json' }
         | 
| 67 | 
            +
                  base = 'http://coordinator.stub/druid/coordinator/v1/'
         | 
| 68 | 
            +
             | 
| 69 | 
            +
                  # Stub all queries to the API
         | 
| 70 | 
            +
                  stub_request(:get, "#{base}leader").with(headers: head)
         | 
| 71 | 
            +
                    .to_return(status: 200, body: responses['leader'], headers: {})
         | 
| 62 72 |  | 
| 63 | 
            -
                   | 
| 64 | 
            -
             | 
| 65 | 
            -
                  #   .to_return(status: 200, body: '{"datasource1":100.0,"datasource2":100.0}',
         | 
| 66 | 
            -
                  #              headers: { 'Content-Type' => 'application/json' })
         | 
| 73 | 
            +
                  stub_request(:get, "#{base}loadstatus").with(headers: head)
         | 
| 74 | 
            +
                    .to_return(status: 200, body: responses['loadstatus'], headers: rhead)
         | 
| 67 75 |  | 
| 68 | 
            -
                   | 
| 69 | 
            -
             | 
| 70 | 
            -
             | 
| 71 | 
            -
                  #              headers: { 'Content-Type' => 'application/json' })
         | 
| 76 | 
            +
                  stub_request(:get, "#{base}loadstatus?simple").with(headers: head)
         | 
| 77 | 
            +
                    .to_return(status: 200, body: responses['loadstatus_simple'],
         | 
| 78 | 
            +
                               headers: rhead)
         | 
| 72 79 |  | 
| 73 | 
            -
                   | 
| 74 | 
            -
             | 
| 75 | 
            -
             | 
| 76 | 
            -
                  #              headers: { 'Content-Type' => 'application/json' })
         | 
| 80 | 
            +
                  stub_request(:get, "#{base}loadstatus?full").with(headers: head)
         | 
| 81 | 
            +
                    .to_return(status: 200, body: responses['loadstatus_full'],
         | 
| 82 | 
            +
                               headers: rhead)
         | 
| 77 83 | 
             
                end
         | 
| 78 84 | 
             
              end
         | 
| 79 85 | 
             
            end
         | 
    
        metadata
    CHANGED
    
    | @@ -1,14 +1,14 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: druid_config
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 0. | 
| 4 | 
            +
              version: 0.2.0
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Angel M Miguel
         | 
| 8 8 | 
             
            autorequire: 
         | 
| 9 9 | 
             
            bindir: bin
         | 
| 10 10 | 
             
            cert_chain: []
         | 
| 11 | 
            -
            date: 2015-10- | 
| 11 | 
            +
            date: 2015-10-30 00:00:00.000000000 Z
         | 
| 12 12 | 
             
            dependencies:
         | 
| 13 13 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 14 14 | 
             
              name: zk
         | 
| @@ -152,12 +152,14 @@ files: | |
| 152 152 | 
             
            - lib/druid_config/entities/node.rb
         | 
| 153 153 | 
             
            - lib/druid_config/entities/rule.rb
         | 
| 154 154 | 
             
            - lib/druid_config/entities/segment.rb
         | 
| 155 | 
            +
            - lib/druid_config/entities/task.rb
         | 
| 155 156 | 
             
            - lib/druid_config/entities/tier.rb
         | 
| 156 157 | 
             
            - lib/druid_config/entities/worker.rb
         | 
| 157 158 | 
             
            - lib/druid_config/util.rb
         | 
| 158 159 | 
             
            - lib/druid_config/version.rb
         | 
| 159 160 | 
             
            - lib/druid_config/zk.rb
         | 
| 160 161 | 
             
            - spec/cluster_spec.rb
         | 
| 162 | 
            +
            - spec/data/druid_responses.yml
         | 
| 161 163 | 
             
            - spec/data_source_spec.rb
         | 
| 162 164 | 
             
            - spec/node_spec.rb
         | 
| 163 165 | 
             
            - spec/spec_helper.rb
         | 
| @@ -187,6 +189,7 @@ specification_version: 4 | |
| 187 189 | 
             
            summary: Fetch Druid info and configure your cluster
         | 
| 188 190 | 
             
            test_files:
         | 
| 189 191 | 
             
            - spec/cluster_spec.rb
         | 
| 192 | 
            +
            - spec/data/druid_responses.yml
         | 
| 190 193 | 
             
            - spec/data_source_spec.rb
         | 
| 191 194 | 
             
            - spec/node_spec.rb
         | 
| 192 195 | 
             
            - spec/spec_helper.rb
         |