druid_config 0.1.0

data/lib/druid_config/entities/worker.rb

@@ -0,0 +1,51 @@
+module DruidConfig
+  module Entities
+    #
+    # Worker class
+    #
+    class Worker
+      # Readers
+      attr_reader :last_completed_task_time, :host, :port, :ip, :capacity,
+                   :version, :running_tasks, :current_capacity_used
+
+      #
+      # Initialize it with received info
+      #
+      # == Parameters:
+      # metadata::
+      #   Hash with returned metadata from Druid
+      #
+      def initialize(metadata)
+        @host, @port = metadata['worker']['host'].split(':')
+        @ip = metadata['worker']['ip']
+        @capacity = metadata['worker']['capacity']
+        @version = metadata['worker']['version']
+        @last_completed_task_time = metadata['lastCompletedTaskTime']
+        @running_tasks = metadata['runningTasks']
+        @capacity_used = metadata['currCapacityUsed']
+      end
+
+      #
+      # Return free capacity
+      #
+      def free
+        @free ||= (capacity - current_capacity_used)
+      end
+
+      #
+      # Return capacity used
+      #
+      def used
+        return 0 unless @capacity && @capacity != 0
+        ((@capacity_used.to_f / @capacity) * 100).round(2)
+      end
+
+      #
+      # Return the uri of the worker
+      #
+      def uri
+        "#{@host}:#{@port}"
+      end
+    end
+  end
+end

data/lib/druid_config/util.rb

@@ -0,0 +1,35 @@
+#
+# Define the versions of the gem.
+#
+module DruidConfig
+  #
+  # Commmon functions for the gem
+  #
+  module Util
+    #
+    # This method is used to protect the Gem to API errors. If a query fails,
+    # the client will be reset and try the query to new coordinator. If it
+    # fails too, a DruidApiError will be launched.
+    #
+    # If the error comes from another point of the code, the Exception
+    # is launched normally
+    #
+    def secure_query
+      return unless block_given?
+      @retries = 0
+      begin
+        yield
+      rescue HTTParty::RedirectionTooDeep => e
+        raise(DruidApiError, e) if @retries > 0
+        @retries += 1
+        reset!
+        retry
+      rescue Errno::ECONNREFUSED => e
+        raise(DruidApiError, e) if @retries > 0
+        @retries += 1
+        reset!
+        retry
+      end
+    end
+  end
+end

data/lib/druid_config/version.rb

@@ -0,0 +1,12 @@
+#
+# Define the versions of the gem.
+#
+module DruidConfig
+  module Version
+    # Version of the gem
+    VERSION = '0.1.0'
+
+    # Base URI foor coordinator queries
+    API_VERSION = 'v1'
+  end
+end

data/lib/druid_config/zk.rb

@@ -0,0 +1,216 @@
+require 'zk'
+require 'rest_client'
+
+module DruidConfig
+  #
+  # Class to connect and get information about nodes in cluster using
+  # Zookeeper
+  #
+  class ZK
+    # Coordinator service
+    COORDINATOR = 'coordinator'
+    OVERLORD = 'overlord'
+    SERVICES = [COORDINATOR, OVERLORD]
+
+    #
+    # Initialize variables and call register
+    #
+    # == Parameters:
+    # uri::
+    #   Uri of zookeper
+    # opts::
+    #   Hash with options:
+    #     - discovery_path: Custom URL of discovery path for Druid
+    #
+    def initialize(uri, opts = {})
+      # Control Zookeper connection
+      @zk = ::ZK.new(uri, chroot: :check)
+      @registry = Hash.new { |hash, key| hash[key] = [] }
+      @discovery_path = opts[:discovery_path] || '/discovery'
+      @watched_services = {}
+      register
+    end
+
+    #
+    # Load the data from Zookeeper
+    #
+    def register
+      $log.info('druid.zk register discovery path') if $log
+      @zk.on_expired_session { register }
+      @zk.register(@discovery_path, only: :child) do
+        $log.info('druid.zk got event on discovery path') if $log
+        check_services
+      end
+      check_services
+    end
+
+    #
+    # Force to close Zookeper connection
+    #
+    def close!
+      $log.info('druid.zk shutting down') if $log
+      @zk.close!
+    end
+
+    #
+    # Return the URI of a random available coordinator.
+    # Poor mans load balancing
+    #
+    def coordinator
+      random_node(COORDINATOR)
+    end
+
+    #
+    # Return the URI of a random available overlord.
+    # Poor mans load balancing
+    #
+    def overlord
+      random_node(OVERLORD)
+    end
+
+    #
+    # Return a random value of a service
+    #
+    # == Parameters:
+    # service::
+    #   String with the name of the service
+    #
+    def random_node(service)
+      return nil if @registry[service].size == 0
+      # Return a random broker from available brokers
+      i = Random.rand(@registry[service].size)
+      @registry[service][i][:uri]
+    end
+
+    #
+    # Register a new service
+    #
+    def register_service(service, brokers)
+      $log.info("druid.zk register", service: service, brokers: brokers) if $log
+      # poor mans load balancing
+      @registry[service] = brokers.shuffle
+    end
+
+    #
+    # Unregister a service
+    #
+    def unregister_service(service)
+      $log.info("druid.zk unregister", service: service) if $log
+      @registry.delete(service)
+      unwatch_service(service)
+    end
+
+    #
+    # Set a watcher for a service
+    #
+    def watch_service(service)
+      return if @watched_services.include?(service)
+      $log.info("druid.zk watch", service: service) if $log
+      watch = @zk.register(watch_path(service), only: :child) do |event|
+        $log.info("druid.zk got event on watch path for", service: service, event: event) if $log
+        unwatch_service(service)
+        check_service(service)
+      end
+      @watched_services[service] = watch
+    end
+
+    #
+    # Unset a service to watch
+    #
+    def unwatch_service(service)
+      return unless @watched_services.include?(service)
+      $log.info("druid.zk unwatch", service: service) if $log
+      @watched_services.delete(service).unregister
+    end
+
+    #
+    # Check current services
+    #
+    def check_services
+      $log.info("druid.zk checking services") if $log
+      zk_services = @zk.children(@discovery_path, watch: true)
+
+      (services - zk_services).each do |service|
+        unregister_service(service)
+      end
+
+      zk_services.each do |service|
+        check_service(service)
+      end
+    end
+
+    #
+    # Verify is a Coordinator is available
+    #
+    # == Parameters:
+    # name::
+    #   String with the name of the coordinator
+    # service::
+    #   String with the service
+    #
+    # == Returns:
+    # URI of the coordinator or false
+    #
+    def verify_node(name, service)
+      $log.info("druid.zk verify", node: name, service: service) if $log
+      info = @zk.get("#{watch_path(service)}/#{name}")
+      node = JSON.parse(info[0])
+      uri = "http://#{node['address']}:#{node['port']}/"
+      check = RestClient::Request.execute(
+        method: :get, url: "#{uri}status",
+        timeout: 5, open_timeout: 5
+      )
+      $log.info("druid.zk verified", uri: uri, sources: check) if $log
+      return uri if check.code == 200
+    rescue
+      return false
+    end
+
+    #
+    # Watch path of a service
+    #
+    def watch_path(service)
+      "#{@discovery_path}/#{service}"
+    end
+
+    #
+    # Check a service
+    #
+    def check_service(service)
+      return if @watched_services.include?(service) ||
+                !SERVICES.include?(service)
+
+      # Start to watch this service
+      watch_service(service)
+
+      known = @registry[service].map { |node| node[:name] }
+      live = @zk.children(watch_path(service), watch: true)
+      new_list = @registry[service].select { |node| live.include?(node[:name]) }
+      $log.info("druid.zk checking", service: service, known: known, live: live, new_list: new_list) if $log
+
+      # verify the new entries to be living brokers
+      (live - known).each do |name|
+        uri = verify_node(name, service)
+        new_list.push(name: name, uri: uri) if uri
+      end
+
+      if new_list.empty?
+        # don't show services w/o active brokers
+        unregister_service(service)
+      else
+        register_service(service, new_list)
+      end
+    end
+
+    #
+    # Get all available services
+    #
+    def services
+      @registry.keys
+    end
+
+    def to_s
+      @registry.to_s
+    end
+  end
+end

data/lib/druid_config.rb

@@ -0,0 +1,39 @@
+# Global library
+require 'httparty'
+
+# Classes
+require 'druid_config/zk'
+require 'druid_config/version'
+require 'druid_config/util'
+require 'druid_config/entities/segment'
+require 'druid_config/entities/worker'
+require 'druid_config/entities/node'
+require 'druid_config/entities/tier'
+require 'druid_config/entities/data_source'
+require 'druid_config/cluster'
+require 'druid_config/client'
+
+# Base namespace of the gem
+module DruidConfig
+  #
+  # Exception class for an error to connect the API
+  #
+  class DruidApiError < StandardError; end
+  
+  # Global client of Druidconfig module
+  @client = nil
+
+  #
+  # Initialize the current client
+  #
+  def self.client=(client)
+    @client = client
+  end
+
+  #
+  # Return initialized client
+  #
+  def self.client
+    @client
+  end
+end

data/spec/cluster_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+require 'pry'
+require 'pry-nav'
+
+# describe DruidConfig::Cluster do
+#   before(:each) do
+#     @cluster = DruidConfig::Cluster.new('localhost', zk_keepalive: true)
+#   end
+
+#   it 'must get the leader' do
+#     expect(@cluster.leader).to eq 'coordinator.stub'
+#   end
+
+#   it 'must get load datasources of a cluster' do
+#     datasources = @cluster.datasources
+
+
+#     # basic = @cluster.load_status
+#     # expect(basic.keys).to eq %w(datasource1 datasource2)
+#     # expect(basic[basic.keys.first]).to eq 100
+
+#     # simple = @cluster.load_status('simple')
+#     # expect(simple.keys).to eq %w(datasource1 datasource2)
+#     # expect(simple[simple.keys.first]).to eq 0
+
+#     # # Use tiers
+#     # simple = @cluster.load_status('full')
+#     # expect(simple.keys).to eq %w(_default_tier hot)
+#     # expect(simple['_default_tier']['datasource1']).to eq 0
+#     # expect(simple['hot']['datasource2']).to eq 0
+#   end
+# end

data/spec/data_source_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+
+describe DruidConfig::Entities::DataSource do
+  before(:each) do
+    @name = 'datasource'
+    @properties = { 'client' => 'side' }
+    @metadata = { 'name' => @name, 'properties' => @properties }
+    @load_status = 100
+  end
+
+  it 'initialize the model based on metadata' do
+    datasource = DruidConfig::Entities::DataSource.new(@metadata, @load_status)
+    expect(datasource.name).to eq @name
+    expect(datasource.properties).to eq @properties
+    expect(datasource.load_status).to eq @load_status
+  end
+end

data/spec/node_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+
+describe DruidConfig::Entities::Node do
+  before(:each) do
+    @host = 'stubbed.cluster:8083'
+    @max_size = 100_000
+    @type = 'historical'
+    @priority = 0
+    @segments = {
+      'datasource_2015-10-22T15:00:00.000Z_2015-10-22T16:00:00.000Z_2015-10-22T15:00:17.214Z' => {
+        'dataSource' => 'datasource',
+        'interval' => '2015-10-22T15:00:00.000Z/2015-10-22T16:00:00.000Z',
+        'version' => '2015-10-22T15:00:17.214Z',
+        'loadSpec' => {},
+        'dimensions' => '',
+        'metrics' => 'events,sum_bytes',
+        'shardSpec' => {
+          'type' => 'linear',
+          'partitionNum' => 0
+        },
+        'binaryVersion' => nil,
+        'size' => 0,
+        'identifier' => 'datasource_2015-10-22T15:00:00.000Z_2015-10-22T16:00:00.000Z_2015-10-22T15:00:17.214Z'
+      }
+    }
+    @size = 50_000
+    @metadata = { 'host' => @host, 'maxSize' => @max_size, 'type' => @type,
+                  'priority' => 0, 'segments' => @segments, 'currSize' => @size }
+    
+    @queue = {
+      'segmentsToLoad' => [],
+      'segmentsToDrop' => []
+    }
+  end
+
+  it 'initialize a Node based on metadata' do
+    datasource = DruidConfig::Entities::Node.new(@metadata, @queue)
+    expect(datasource.host).to eq @host.split(':').first
+    expect(datasource.uri).to eq @host
+    expect(datasource.max_size).to eq @max_size
+    expect(datasource.type).to eq @type.to_sym
+    expect(datasource.priority).to eq @priority
+    expect(datasource.size).to eq @size
+    expect(datasource.segments_to_load).to eq []
+    expect(datasource.segments_to_drop).to eq []
+  end
+
+  it 'calculate free space' do
+    datasource = DruidConfig::Entities::Node.new(@metadata, @queue)
+    expect(datasource.free).to eq(@max_size - @size)
+  end
+
+  it 'calculate percentage of used space' do
+    datasource = DruidConfig::Entities::Node.new(@metadata, @queue)
+    expect(datasource.used_percent).to eq((@size.to_f / @max_size) * 100)
+  end
+
+  it 'return 0 when max size is 0' do
+    datasource =
+      DruidConfig::Entities::Node.new(@metadata.merge('maxSize' => 0), @queue)
+    expect(datasource.used_percent).to eq 0
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,79 @@
+require 'druid_config'
+require 'webmock/rspec'
+
+# Mock Druid
+ENV['MOCK_DRUID'] ||= 'false'
+
+if ENV['MOCK_DRUID'] == 'true'
+  # Disable external connections
+  WebMock.disable_net_connect!(allow_localhost: true)
+end
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # Use color in STDOUT
+  config.color = true
+  # Use color not only in STDOUT but also in pagers and files
+  config.tty = true
+  # Use the specified formatter
+  config.formatter = :documentation # :progress, :html, :textmate
+
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  #
+  # Mock druid API queries
+  #
+  config.before(:each) do
+    if ENV['MOCK_DRUID'] == 'true'
+      # Stub DruidConfig::Client to ignore Zookeeper.
+      # TODO: We must improve it!!!
+      class ClientStub
+        def coordinator
+          'coordinator.stub/'
+        end
+      end
+      allow(DruidConfig).to receive(:client) { ClientStub.new }
+
+      # Stub queries
+      # ----------------------------------
+      
+      # Our scenario:
+      # leader: coordinator.stub
+      # datasources: datasource1, datasource2
+      # tiers: _default_tier, hot
+      # stub_request(:get, 'http://coordinator.stub/druid/coordinator/v1/leader')
+      #   .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' })
+      #   .to_return(status: 200, body: 'coordinator.stub', headers: {})
+
+      # stub_request(:get, 'http://coordinator.stub/druid/coordinator/v1/loadstatus')
+      #   .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' })
+      #   .to_return(status: 200, body: '{"datasource1":100.0,"datasource2":100.0}',
+      #              headers: { 'Content-Type' => 'application/json' })
+
+      # stub_request(:get, 'http://coordinator.stub/druid/coordinator/v1/loadstatus?simple')
+      #   .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' })
+      #   .to_return(status: 200, body: '{"datasource1":0,"datasource2":0}',
+      #              headers: { 'Content-Type' => 'application/json' })
+
+      # stub_request(:get, 'http://coordinator.stub/druid/coordinator/v1/loadstatus?full')
+      #   .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' })
+      #   .to_return(status: 200, body: '{"_default_tier":{"datasource1":0}, "hot":{"datasource2":0}}',
+      #              headers: { 'Content-Type' => 'application/json' })
+    end
+  end
+end