druid_config 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a03eb8b4f7df4223fe098516a23443beac52be00
-  data.tar.gz: 230450c3d105e2327c681f1bc1beb225926361be
+  metadata.gz: 34faeb23be9dfd54181862f172bb5db8b794f489
+  data.tar.gz: 58c09b7c65c33be02e1b2cf0f9ca8eaa3754666f
 SHA512:
-  metadata.gz: 151efb283126cbf585549a0e1d0a7f766f48c51e3f21e18ab373c34d59a247c64fbcc1188157ccaadb7b5940ccdcc480c24b87f33a2f0e890e019be817b9bcab
-  data.tar.gz: db35a7cc044f47bb4de83ea1467e35186389471485f71f48e3f4363087dc8f9326150a0df93a7844d901b804bf38348d51ab2cfb452a2aa708240cbd08a5ba76
+  metadata.gz: 64062d2ad8ec0abc2487a003feb746d175aaf6cc8b47b703fbc1544e1a59bd1089209c6d61d164678cbe469884ebefff97f0114cec7884bc63daea73b44df45f
+  data.tar.gz: 7149d8613c814c63ae21ccae898a0205dcbe479a5b0973f757d462a4d8b4d47a5ebefff5fd1cda6527b73e61222c68ff221dc893523735ae33e579617422b129

data/README.md

@@ -44,6 +44,8 @@ Call methods defined in `DruidConfig::Cluster` to access to the data. To get mor
 * `realtimes`: realtime nodes
 * `workers`: worker nodes
 * `physical_workers`: array of URIs of worker nodes
+* `running_tasks`, `pending_tasks`, `waiting_tasks`, `complete_tasks`: tasks in the cluster based in their status
+* `task`: load a task based on an identifier
 * `services`: Hash with physical nodes and the services they are running
 
 ## Entities
@@ -55,6 +57,7 @@ Some methods return an instance of an `Entity` class. These entities provide mul
 * [Tier](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/tier.rb)
 * [Node](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/node.rb)
 * [Worker](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/worker.rb)
+* [Task](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/task.rb)
 
 ## Exceptions
 

data/lib/druid_config.rb

@@ -1,11 +1,17 @@
 # Global library
 require 'httparty'
+require 'iso8601'
+require 'delegate'
+require 'json'
+require 'net/http'
 
 # Classes
 require 'druid_config/zk'
 require 'druid_config/version'
 require 'druid_config/util'
 require 'druid_config/entities/segment'
+require 'druid_config/entities/rule'
+require 'druid_config/entities/rule_collection'
 require 'druid_config/entities/task'
 require 'druid_config/entities/worker'
 require 'druid_config/entities/node'

data/lib/druid_config/cluster.rb

@@ -158,6 +158,17 @@ module DruidConfig
       datasources.select { |el| el.name == datasource }
     end
 
+    #
+    # Return default datasource. This datasource hasn't got metadata
+    # associated. It's only used to read and apply default rules.
+    #
+    def default_datasource
+      DruidConfig::Entities::DataSource.new(
+        { 'name' => DruidConfig::Entities::DataSource::DEFAULT_DATASOURCE,
+          'properties' => {} },
+        {})
+    end
+
     # Rules
     # -----------------
     
@@ -165,9 +176,16 @@ module DruidConfig
     # Return the rules applied to a cluster
     #
     def rules
+      rules = DruidConfig::Entities::RuleCollection.new
       secure_query do
-        self.class.get('/rules')
+        self.class.get('/rules').each do |datasource, ds_rules|
+          ds_rules.each do |rule|
+            rules << DruidConfig::Entities::Rule.parse(rule, datasource)
+          end
+        end
       end
+      # Return initialized rules
+      rules
     end
 
     # Tiers

data/lib/druid_config/entities/data_source.rb

@@ -12,6 +12,9 @@ module DruidConfig
 
       attr_reader :name, :properties, :load_status
 
+      # Name of default datasource
+      DEFAULT_DATASOURCE = '_default'.freeze
+
       #
       # Initialize a DataSource
       #
@@ -75,17 +78,76 @@ module DruidConfig
       # Rules
       # -----------------
       def rules(params = '')
+        return @rules if @rules
+        @rules = DruidConfig::Entities::RuleCollection.new
         secure_query do
-          self.class.get("/rules/#{@name}?#{params}")
+          self.class.get("/rules/#{@name}?#{params}").each do |rule|
+            @rules << DruidConfig::Entities::Rule.parse(rule)
+          end
+        end
+        @rules
+      end
+
+      #
+      # Apply given rules to the datasource
+      #
+      # == Paremeters:
+      # rules::
+      #   RuleCollection of rules
+      #
+      # == Returns:
+      # Boolean indicating the status of the request
+      #
+      def update_rules(new_rules)
+        if post_rules(new_rules)
+          @rules = new_rules
+          true
+        else
+          false
         end
       end
 
+      #
+      # Save current rules
+      #
+      # == Returns:
+      # Boolean indicating the status of the request
+      #
+      def save_rules
+        post_rules(rules)
+      end
+
       def history_rules(interval)
         secure_query do
           self.class.get("/rules/#{@name}/history"\
                          "?interval=#{interval}")
         end
       end
+
+      private
+
+      #
+      # Save rules of this data source
+      #
+      # == Paremeters:
+      # rules::
+      #   RuleCollection of rules
+      #
+      # == Returns:
+      # Boolean indicating the status of the request
+      #
+      def post_rules(new_rules)
+        fail(ArgumentError, 'Rules must be a RuleCollection instance') unless
+          new_rules.is_a?(RuleCollection)
+        uri = URI("#{self.class.base_uri}/rules/#{name}")
+        http = Net::HTTP.new(uri.host, uri.port)
+        request = Net::HTTP::Post.new(uri.request_uri)
+        request['Content-Type'] = 'application/json'
+        request.body = new_rules.map(&:to_h).to_json
+        response = http.request(request)
+        # Check statys
+        response.code.to_i == 200
+      end
     end
   end
 end

data/lib/druid_config/entities/node.rb

@@ -60,7 +60,8 @@ module DruidConfig
       # Calculate free space
       #
       def free
-        max_size - size
+        return @free if @free
+        @free = (max_size - size) > 0 ? (max_size - size) : 0
       end
 
       #

data/lib/druid_config/entities/rule.rb

@@ -0,0 +1,164 @@
+module DruidConfig
+  module Entities
+    #
+    # Rule class
+    #
+    class Rule
+      # Variables
+      attr_reader :datasource, :type, :time_type, :replicants, :period,
+                  :interval
+
+      # Identifier for type
+      FOREVER_DRUID_STRING = 'Forever'
+      INTERVAL_DRUID_STRING = 'ByInterval'
+      PERIOD_DRUID_STRING = 'ByPeriod'
+
+      #
+      # Parse data from a Druid API response an initialize an object of
+      # Rule class
+      #
+      # == Parameters:
+      # datasource::
+      #   String with the name of the datsource
+      # data::
+      #   Hash provided by the API
+      #
+      # == Returns:
+      # Rule instance
+      #
+      def self.parse(data, datasource = nil)
+        type, time_type = detect_type(data['type'])
+        options = { replicants: data['tieredReplicants'] }
+        options.merge!(datasource: datasource) if datasource
+        if time_type == :period
+          options.merge!(period: data['period'])
+        elsif time_type == :interval
+          options.merge!(interval: data['interval'])
+        end
+        # Instance the class
+        new(type, time_type, options)
+      end
+
+      #
+      # Initialize a Rule object. This constructor accepts a Hash with
+      # format defined in:
+      #   http://druid.io/docs/latest/operations/rule-configuration.html
+      #
+      # == Parameters:
+      # datasource::
+      #   String with the name of the data source
+      # type::
+      #   Type of the rule, it can be :drop or :load
+      # time_type::
+      #   Time reference. It can be :forever, :period or :interval
+      # options::
+      #   Hash with extra data to the rules.
+      #     - replicants: Hash with format
+      #         { 'tier' => NumberOfReplicants, 'tier' => ... }
+      #     - period: String with a period in ISO8601 format.
+      #               Only available when type is :period.
+      #     - interval: String with a interval in ISO8601 format.
+      #                 Only available when type is :interval.
+      #     - datasource: Name of the datasource
+      #
+      def initialize(type, time_type, options = {})
+        @type = type
+        @time_type = time_type
+        @datasource = options[:datasource]
+        @replicants = options[:replicants]
+        if period?
+          @period = ISO8601::Duration.new(options[:period])
+        elsif interval?
+          # TODO: https://github.com/arnau/ISO8601/issues/15
+          @interval = options[:interval]
+        end
+      end
+
+      #
+      # Functions to check the rule type
+      #
+      %w(drop load).each do |rule|
+        define_method("#{rule}?") do
+          @type == rule.to_sym
+        end
+      end
+
+      #
+      # Functions to check how rule time is defined
+      #
+      %w(interval forever period).each do |time|
+        define_method("#{time}?") do
+          @time_type == time.to_sym
+        end
+      end
+
+      #
+      # Return the rule as Hash format
+      #
+      # == Returns:
+      # Hash
+      #
+      def to_h
+        base = { type: type_to_druid }
+        base.merge!(tieredReplicants: @replicants) if @replicants
+        if period?
+          base.merge(period: @period.to_s)
+        elsif interval?
+          base.merge(interval: @interval.to_s)
+        else
+          base
+        end
+      end
+
+      #
+      # Return the rule as valid JSON for Druid
+      #
+      # == Returns:
+      # JSON String
+      #
+      def to_json
+        to_h.to_json
+      end
+
+      #
+      # Detect the type of the rule based on 'type' field. This method will
+      # detect if is a drop/load rule and how it defines time.
+      #
+      # == Parameters:
+      # type_to_parse::
+      #   String with the content of type field
+      #
+      def self.detect_type(type_to_parse)
+        type = type_to_parse.starts_with?('drop') ? :drop : :load
+        time_type = case type_to_parse.gsub(type.to_s, '')
+                    when INTERVAL_DRUID_STRING
+                      :interval
+                    when FOREVER_DRUID_STRING
+                      :forever
+                    when PERIOD_DRUID_STRING
+                      :period
+                    end
+        [type, time_type]
+      end
+
+      private
+
+      #
+      # Convert the type to an String Druid can identify
+      #
+      # == Returns:
+      # String with the type of the rule
+      #
+      def type_to_druid
+        time = if period?
+                 PERIOD_DRUID_STRING
+               elsif interval?
+                 INTERVAL_DRUID_STRING
+               else
+                 FOREVER_DRUID_STRING
+               end
+        "#{@type}#{time}"
+      end
+    end
+  end
+end

data/lib/druid_config/entities/rule_collection.rb

@@ -0,0 +1,65 @@
+module DruidConfig
+  module Entities
+    #
+    # Rule set of a data source
+    #
+    class RuleCollection < Array
+      #
+      # Check the consistency of the rules. For example, if you define the
+      # following rules:
+      #
+      # 1 - LOAD ByPeriod PT12M replicants -> { _default_tier => 1 }
+      # 2 - LOAD ByPeriod PT3M replicants -> { _default_tier => 1 }
+      #
+      # The second rule never be applied.
+      #
+      # This method don't raise any exception and your Druid installation will
+      # work unless there are an inconsistence. The purpose of this method is
+      # to advise you could be wrong about rules
+      #
+      # == Returns:
+      # A boolean. True when the rules are consistent.
+      #
+      def consistent?
+        # TODO: implement this method
+        true
+      end
+
+      #
+      # Return the collection of rules as a valid JSON for Druid
+      #
+      # == Parameters:
+      # include_datasources::
+      #   True if you want to include the name of the datasources as keys
+      #   (False by default)
+      #
+      # == Returns:
+      # JSON String
+      #
+      def to_json(include_datasources = false)
+        return to_json_with_datasources if include_datasources
+        map(&:to_h).to_json
+      end
+
+      private
+
+      #
+      # Return a JSON string with the datasources and the rules associated to
+      # them
+      #
+      # == Returns:
+      # JSON string with format:
+      #   { 'datasource' => [ { "type" => ... }, ...], 'datasource2' => [...] }
+      #
+      def to_json_with_datasources
+        rules_with_ds = {}
+        map do |rule|
+          rules_with_ds[rule.datasource] ||= []
+          rules_with_ds[rule.datasource] << rule.to_h
+        end
+        rules_with_ds.to_json
+      end
+    end
+  end
+end
+    

data/lib/druid_config/entities/tier.rb

@@ -25,7 +25,8 @@ module DruidConfig
       end
 
       def free
-        @free ||= (max_size - size)
+        return @free if @free
+        @free = (max_size - size) > 0 ? (max_size - size) : 0
       end
 
       def used_percent

data/lib/druid_config/entities/worker.rb

@@ -37,7 +37,8 @@ module DruidConfig
       # Return free capacity
       #
       def free
-        @free ||= (capacity - capacity_used)
+        return @free if @free
+        @free = (capacity - capacity_used) > 0 ? (capacity - capacity_used) : 0
       end
 
       #

data/lib/druid_config/validators/rule_validator.rb

@@ -0,0 +1,10 @@
+module DruidConfig
+  module Validators
+    #
+    # Validator of rule class.
+    #
+    class RuleValidator
+
+    end
+  end
+end

data/lib/druid_config/version.rb

@@ -4,7 +4,7 @@
 module DruidConfig
   module Version
     # Version of the gem
-    VERSION = '0.2.0'
+    VERSION = '0.3.0'
 
     # Base URI foor coordinator queries
     API_VERSION = 'v1'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: druid_config
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Angel M Miguel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-30 00:00:00.000000000 Z
+date: 2015-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: zk
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.8.0
+- !ruby/object:Gem::Dependency
+  name: iso8601
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.7
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -151,11 +165,13 @@ files:
 - lib/druid_config/entities/data_source.rb
 - lib/druid_config/entities/node.rb
 - lib/druid_config/entities/rule.rb
+- lib/druid_config/entities/rule_collection.rb
 - lib/druid_config/entities/segment.rb
 - lib/druid_config/entities/task.rb
 - lib/druid_config/entities/tier.rb
 - lib/druid_config/entities/worker.rb
 - lib/druid_config/util.rb
+- lib/druid_config/validators/rule_validator.rb
 - lib/druid_config/version.rb
 - lib/druid_config/zk.rb
 - spec/cluster_spec.rb