druid_config 0.1.0

data/README.md

@@ -0,0 +1,68 @@
+# DruidConfig
+
+DruidConfig is a gem to access the information about Druid cluster status. You can check a node capacity, number of segments, tiers... It uses [zookeeper](https://zookeeper.apache.org/) to get coordinator and overlord URIs.
+
+To use in your application, add this line to your Gemfile:
+
+```ruby
+require 'druid_config'
+```
+
+# Initialization
+
+`Cluster` is the base class to perform queries. To initialize it send the Zookeeper URI and options as arguments:
+
+```ruby
+cluster = DruidConfig::Cluster.new(zookeeper_uri, options)
+```
+
+Available options:
+* discovery_path: string with the discovery path of druid inside Zookeeper directory structure.
+
+# Usage
+
+Call methods defined in `DruidConfig::Cluster` to access to the data. To get more information about data returned in methods, check [Druid documentation](http://druid.io/docs/0.8.1/design/coordinator.html).
+
+* `leader`: leader
+* `load_status`: load status
+* `load_status`: load queue
+* `metadata_datasources`: Hash with metadata of datasources
+* `metadata_datasources_segments`: Hash with metadata of segments
+* `datasources`: all data sources
+* `datasource`: a concrete data source
+* `rules`: all rules defined in the cluster
+* `tiers`: tiers
+* `servers` or `nodes`: all nodes of the cluster
+* `physical_servers` or `physical_nodes`: array of URIs of nodes
+* `historicals`: historical nodes
+* `realtimes`: realtime nodes
+* `workers`: worker nodes
+* `physical_workers`: array of URIs of worker nodes
+* `services`: Hash with physical nodes and the services they are running
+
+## Entities
+
+Some methods return an instance of an `Entity` class. These entities provide multiple methods to access data. Defined entities are inside `druid_config/entities` folder.
+
+* [DataSource]()
+* [Node]()
+* [Segment]()
+* [Tier]()
+* [Worker]()
+
+## Exceptions
+
+Sometimes the Gem can't access to Druid API. In this case, the gem automatically will reset the Zookeeper connection and retry the query. If second query fails too, a `DruidApiError` exception will be raised.
+
+# Collaborate
+
+To contribute DruidConfig:
+
+* Create an issue with the contribution: bug, enhancement or feature
+* Fork the repository and make all changes you need
+* Write test on new changes
+* Create a pull request when you finish
+
+# License
+
+DruidConfig gem is released under the Affero GPL license. Copyright [redBorder](http://redborder.net)

data/lib/druid_config/client.rb

@@ -0,0 +1,46 @@
+module DruidConfig
+  #
+  # Class to initialize the connection to Zookeeper
+  #
+  class Client
+    attr_reader :zk, :zookeeper, :opts
+
+    #
+    # Initialize Zookeeper connection
+    #
+    def initialize(zookeeper, opts = {})
+      @zookeeper = :zk
+      @opts = opts
+      @zk = ZK.new(zookeeper, opts)
+    end
+
+    #
+    # Get the URL of a coordinator
+    #
+    def coordinator
+      zk.coordinator
+    end
+
+    #
+    # Get the URI of a overlord
+    #
+    def overlord
+      zk.overlord
+    end
+
+    #
+    # Close the client
+    #
+    def close!
+      zk.close!
+    end
+
+    #
+    # Reset the client
+    #
+    def reset!
+      close!
+      @zk = ZK.new(@zookeeper, @opts)
+    end
+  end
+end

data/lib/druid_config/cluster.rb

@@ -0,0 +1,324 @@
+module DruidConfig
+  #
+  # Class to initialize the connection to Zookeeper
+  #
+  class Cluster
+    # HTTParty Rocks!
+    include HTTParty
+    include DruidConfig::Util
+
+    #
+    # Initialize the client to perform the queries
+    #
+    # == Parameters:
+    # zk_uri::
+    #   String with URI or URIs (sparated by comma) of Zookeeper
+    # options::
+    #   Hash with options:
+    #     - discovery_path: String with the discovery path of Druid
+    #
+    def initialize(zk_uri, options)
+      # Initialize the Client
+      DruidConfig.client = DruidConfig::Client.new(zk_uri, options)
+
+      # Used to check the number of retries on error
+      @retries = 0
+
+      # Update the base uri to perform queries
+      self.class.base_uri(
+        "#{DruidConfig.client.coordinator}"\
+        "druid/coordinator/#{DruidConfig::Version::API_VERSION}")
+    end
+
+    #
+    # Close connection with zookeeper
+    #
+    def close!
+      DruidConfig.client.close!
+    end
+
+    #
+    # Reset the client
+    #
+    def reset!
+      DruidConfig.client.reset!
+      self.class.base_uri(
+        "#{DruidConfig.client.coordinator}"\
+        "druid/coordinator/#{DruidConfig::Version::API_VERSION}")
+    end
+
+    # ------------------------------------------------------------
+    # Queries!
+    # ------------------------------------------------------------
+    
+    #
+    # The following methods are referenced to Druid API. To check the
+    # funcionality about it, please go to Druid documentation:
+    #
+    # http://druid.io/docs/0.8.1/design/coordinator.html
+    #
+
+    # Coordinator
+    # -----------------
+ 
+    #
+    # Return the leader of the Druid cluster
+    #
+    def leader
+      secure_query do
+        self.class.get('/leader').body
+      end
+    end
+
+    #
+    # Load status of the cluster
+    #
+    def load_status(params = '')
+      secure_query do
+        self.class.get("/loadstatus?#{params}")
+      end
+    end
+
+    #
+    # Load queue of the cluster
+    #
+    def load_queue(params = '')
+      secure_query do
+        self.class.get("/loadqueue?#{params}")
+      end
+    end
+
+    # Metadata
+    # -----------------
+    
+    #
+    # Return a Hash with metadata of datasources
+    #
+    def metadata_datasources(params = '')
+      secure_query do
+        self.class.get("/metadata/datasources?#{params}")
+      end
+    end
+
+    alias_method :mt_datasources, :metadata_datasources
+
+    #
+    # Return a Hash with metadata of segments
+    #
+    # == Parameters:
+    # data_source::
+    #   String with the name of the data source
+    # segment::
+    #   (Optional) Segment to search
+    #
+    def metadata_datasources_segments(data_source, segment = '')
+      end_point = "/metadata/datasources/#{data_source}/segments"
+      secure_query do
+        if segment.empty? || segment == 'full'
+          self.class.get("#{end_point}?#{params}")
+        else
+          self.class.get("#{end_point}/#{params}")
+        end
+      end
+    end
+
+    alias_method :mt_datasources_segments, :metadata_datasources_segments
+
+    # Data sources
+    # -----------------
+    
+    #
+    # Return all datasources
+    #
+    # == Returns:
+    # Array of Datasource initialized.
+    #
+    def datasources
+      datasource_status = load_status
+      secure_query do
+        self.class.get('/datasources?full').map do |data|
+          DruidConfig::Entities::DataSource.new(
+            data,
+            datasource_status.select { |k, _| k == data['name'] }.values.first)
+        end
+      end
+    end
+
+    #
+    # Return a unique datasource
+    #
+    # == Parameters:
+    # datasource:
+    #   String with the data source name
+    #
+    # == Returns:
+    # DataSource instance
+    #
+    def datasource(datasource)
+      datasources.select { |el| el.name == datasource }
+    end
+
+    # Rules
+    # -----------------
+    
+    #
+    # Return the rules applied to a cluster
+    #
+    def rules
+      secure_query do
+        self.class.get('/rules')
+      end
+    end
+
+    # Tiers
+    # -----------------
+ 
+    #
+    # Return all tiers defined in the cluster
+    #
+    # == Returns:
+    # Array of Tier instances
+    #
+    def tiers
+      current_nodes = servers
+      # Initialize tiers
+      secure_query do
+        current_nodes.map(&:tier).uniq.map do |tier|
+          DruidConfig::Entities::Tier.new(
+            tier,
+            current_nodes.select { |node| node.tier == tier })
+        end
+      end
+    end
+
+    # Servers
+    # -----------------
+
+    #
+    # Return all nodes of the cluster
+    #
+    # == Returns:
+    # Array of node Objects
+    #
+    def servers
+      secure_query do
+        queue = load_queue('full')
+        self.class.get('/servers?full').map do |data|
+          DruidConfig::Entities::Node.new(
+            data,
+            queue.select { |k, _| k == data['host'] }.values.first)
+        end
+      end
+    end
+
+    #
+    # URIs of the physical servers in the cluster
+    #
+    # == Returns:
+    # Array of strings
+    #
+    def physical_servers
+      secure_query do
+        @physical_servers ||= servers.map(&:host).uniq
+      end
+    end
+
+    alias_method :nodes, :servers
+    alias_method :physical_nodes, :physical_servers
+
+    #
+    # Returns only historial nodes
+    #
+    # == Returns:
+    # Array of Nodes
+    #
+    def historicals
+      servers.select { |node| node.type == :historical }
+    end
+
+    #
+    # Returns only realtime
+    #
+    # == Returns:
+    # Array of Nodes
+    #
+    def realtimes
+      servers.select { |node| node.type == :realtime }
+    end
+
+    #
+    # Return all Workers (MiddleManager) of the cluster
+    #
+    # == Returns:
+    # Array of Workers
+    #
+    def workers
+      # Stash the base_uri
+      stash_uri
+      self.class.base_uri(
+        "#{DruidConfig.client.overlord}"\
+        "druid/indexer/#{DruidConfig::Version::API_VERSION}")
+      workers = []
+      # Perform a query
+      begin
+        secure_query do
+          workers = self.class.get('/workers').map do |worker|
+            DruidConfig::Entities::Worker.new(worker)
+          end
+        end
+      ensure
+        # Recover it
+        pop_uri
+      end
+      # Return
+      workers
+    end
+
+    #
+    # URIs of the physical workers in the cluster
+    #
+    def physical_workers
+      @physical_workers ||= workers.map(&:host).uniq
+    end
+
+    # Services
+    # -----------------
+    
+    #
+    # Availabe services in the cluster
+    #
+    # == Parameters:
+    # Array of Hash with the format:
+    #   { server: [ services ], server2: [ services ], ... }
+    #
+    def services
+      return @services if @services
+      services = {}
+      physical_nodes.each { |node| services[node] = [] }
+      # Load services
+      realtimes.map(&:host).uniq.each { |r| services[r] << :realtime }
+      historicals.map(&:host).uniq.each { |r| services[r] << :historical }
+      physical_workers.each { |w| services[w] << :middleManager }
+      # Return nodes
+      @services = services
+    end
+
+    private
+
+    #
+    # Stash current base_uri
+    #
+    def stash_uri
+      @uri_stack ||= []
+      @uri_stack.push self.class.base_uri
+    end
+
+    #
+    # Pop next base_uri
+    #
+    def pop_uri
+      return if @uri_stack.nil? || @uri_stack.empty?
+      self.class.base_uri(@uri_stack.pop)
+    end
+  end
+end

data/lib/druid_config/entities/data_source.rb

@@ -0,0 +1,78 @@
+module DruidConfig
+  #
+  # Module of info
+  #
+  module Entities
+    #
+    # Init a DataSource
+    class DataSource
+      # HTTParty Rocks!
+      include HTTParty
+
+      attr_reader :name, :properties, :load_status
+
+      #
+      # Initialize a DataSource
+      #
+      def initialize(metadata, load_status)
+        @name = metadata['name']
+        @properties = metadata['properties']
+        @load_status = load_status
+        # Set end point for HTTParty
+        self.class.base_uri(
+          "#{DruidConfig.client.coordinator}"\
+          "druid/coordinator/#{DruidConfig::Version::API_VERSION}")
+      end
+
+      #
+      # The following methods are referenced to Druid API. To check the
+      # funcionality about it, please go to Druid documentation:
+      #
+      # http://druid.io/docs/0.8.1/design/coordinator.html
+      #
+
+      def info(params = '')
+        @info ||= self.class.get("/datasources/#{@name}?#{params}")
+      end
+
+      # Intervals
+      # -----------------
+      def intervals(params = '')
+        self.class.get("/datasources/#{@name}/intervals?#{params}")
+      end
+
+      def interval(interval, params = '')
+        self.class.get("/datasources/#{@name}/intervals/#{interval}"\
+                       "?#{params}")
+      end
+
+      # Segments and Tiers
+      # -----------------
+      def segments
+        @segments ||=
+          self.class.get("/datasources/#{@name}/segments?full").map do |s|
+            DruidConfig::Entities::Segment.new(s)
+          end
+      end
+
+      def segment(segment)
+        segments.select { |s| s.id == segment }
+      end
+
+      def tiers
+        info['tiers']
+      end
+
+      # Rules
+      # -----------------
+      def rules(params = '')
+        self.class.get("/rules/#{@name}?#{params}")
+      end
+
+      def history_rules(interval)
+        self.class.get("/rules/#{@name}/history"\
+                       "?interval=#{interval}")
+      end
+    end
+  end
+end

data/lib/druid_config/entities/node.rb

@@ -0,0 +1,74 @@
+module DruidConfig
+  module Entities
+    #
+    # Node class
+    #
+    class Node
+      # HTTParty Rocks!
+      include HTTParty
+
+      # Readers
+      attr_reader :host, :port, :max_size, :type, :tier, :priority, :size,
+                  :segments, :segments_to_load, :segments_to_drop,
+                  :segments_to_load_size, :segments_to_drop_size
+
+      #
+      # Initialize it with received info
+      #
+      # == Parameters:
+      # metadata::
+      #   Hash with the data of the node given by a Druid API query
+      # queue::
+      #   Hash with segments to load
+      #
+      def initialize(metadata, queue)
+        @host, @port = metadata['host'].split(':')
+        @max_size = metadata['maxSize']
+        @type = metadata['type'].to_sym
+        @tier = metadata['tier']
+        @priority = metadata['priority']
+        @size = metadata['currSize']
+        @segments = metadata['segments'].map do |_, sdata|
+          DruidConfig::Entities::Segment.new(sdata)
+        end
+        if queue.nil?
+          @segments_to_load, @segments_to_drop = [], []
+          @segments_to_load_size, @segments_to_drop_size = 0, 0
+        else
+          @segments_to_load = queue['segmentsToLoad'].map do |segment|
+            DruidConfig::Entities::Segment.new(segment)
+          end
+          @segments_to_drop = queue['segmentsToDrop'].map do |segment|
+            DruidConfig::Entities::Segment.new(segment)
+          end
+          @segments_to_load_size = @segments_to_load.map(&:size).reduce(:+)
+          @segments_to_drop_size = @segments_to_drop.map(&:size).reduce(:+)
+        end
+      end
+
+      alias_method :used, :size
+
+      #
+      # Calculate the percent of used space
+      #
+      def used_percent
+        return 0 unless max_size && max_size != 0
+        ((size.to_f / max_size) * 100).round(2)
+      end
+
+      #
+      # Calculate free space
+      #
+      def free
+        max_size - size
+      end
+
+      #
+      # Return the URI of this node
+      #
+      def uri
+        "#{@host}:#{@port}"
+      end
+    end
+  end
+end

data/lib/druid_config/entities/segment.rb

@@ -0,0 +1,60 @@
+module DruidConfig
+  module Entities
+    #
+    # Segment class
+    #
+    class Segment
+      # Readers
+      attr_reader :id, :interval, :version, :load_spec, :dimensions, :metrics,
+                  :shard_spec, :binary_version, :size
+
+      #
+      # Initialize it with received info
+      #
+      # == Parameters:
+      # metadata::
+      #   Hash with returned metadata from Druid
+      #
+      def initialize(metadata)
+        @id = metadata['identifier']
+        @interval = metadata['interval'].split('/').map { |t| Time.parse t }
+        @version = Time.parse metadata['version']
+        @load_spec = metadata['loadSpec']
+        @dimensions = metadata['dimensions'].split(',').map(&:to_sym)
+        @metrics = metadata['metrics'].split(',').map(&:to_sym)
+        @shard_spec = metadata['shardSpec']
+        @binary_version = metadata['binaryVersion']
+        @size = metadata['size']
+      end
+
+      #
+      # Return direct link to the store
+      #
+      # == Returns:
+      # String with the URI
+      #
+      def store_uri
+        return '' if load_spec.empty?
+        "s3://#{load_spec['bucket']}/#{load_spec['key']}"
+      end
+
+      #
+      # Return the store type
+      #
+      # == Returns:
+      # Store type as symbol
+      #
+      def store_type
+        return nil if load_spec.empty?
+        load_spec['type'].to_sym
+      end
+
+      #
+      # By default, show the identifier in To_s
+      #
+      def to_s
+        @id
+      end
+    end
+  end
+end

data/lib/druid_config/entities/tier.rb

@@ -0,0 +1,66 @@
+module DruidConfig
+  module Entities
+    #
+    # Tier class
+    #
+    class Tier
+      # Readers
+      attr_reader :name, :nodes
+      
+      def initialize(name, nodes)
+        @name = name
+        @nodes = nodes
+      end
+
+      alias_method :servers, :nodes
+
+      def size
+        @size ||= nodes.map(&:size).inject(:+)
+      end
+
+      alias_method :used, :size
+
+      def max_size
+        @max_size ||= nodes.map(&:max_size).inject(:+)
+      end
+
+      def free
+        @free ||= (max_size - size)
+      end
+
+      def used_percent
+        return 0 unless max_size && max_size != 0
+        ((size.to_f / max_size) * 100).round(2)
+      end
+
+      def historicals
+        nodes.select { |node| node.type == :historical }
+      end
+
+      def segments
+        @segments ||= nodes.map(&:segments)
+                      .flatten.sort_by { |seg| seg.interval.first }
+      end
+
+      def segments_to_load
+        @segments_to_load ||=
+          nodes.map { |node| node.segments_to_load.count }.inject(:+)
+      end
+
+      def segments_to_drop
+        @segments_to_drop ||=
+          nodes.map { |node| node.segments_to_drop.count }.inject(:+)
+      end
+
+      def segments_to_load_size
+        @segments_to_load_size ||=
+          nodes.map(&:segments_to_load_size).reduce(:+)
+      end
+
+      def segments_to_drop_size
+        @segments_to_drop_size ||=
+          nodes.map(&:segments_to_drop_size).reduce(:+)
+      end
+    end
+  end
+end