druid_config 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,51 @@
1
+ module DruidConfig
2
+ module Entities
3
+ #
4
+ # Worker class
5
+ #
6
+ class Worker
7
+ # Readers
8
+ attr_reader :last_completed_task_time, :host, :port, :ip, :capacity,
9
+ :version, :running_tasks, :current_capacity_used
10
+
11
+ #
12
+ # Initialize it with received info
13
+ #
14
+ # == Parameters:
15
+ # metadata::
16
+ # Hash with returned metadata from Druid
17
+ #
18
+ def initialize(metadata)
19
+ @host, @port = metadata['worker']['host'].split(':')
20
+ @ip = metadata['worker']['ip']
21
+ @capacity = metadata['worker']['capacity']
22
+ @version = metadata['worker']['version']
23
+ @last_completed_task_time = metadata['lastCompletedTaskTime']
24
+ @running_tasks = metadata['runningTasks']
25
+ @capacity_used = metadata['currCapacityUsed']
26
+ end
27
+
28
+ #
29
+ # Return free capacity
30
+ #
31
+ def free
32
+ @free ||= (capacity - current_capacity_used)
33
+ end
34
+
35
+ #
36
+ # Return capacity used
37
+ #
38
+ def used
39
+ return 0 unless @capacity && @capacity != 0
40
+ ((@capacity_used.to_f / @capacity) * 100).round(2)
41
+ end
42
+
43
+ #
44
+ # Return the uri of the worker
45
+ #
46
+ def uri
47
+ "#{@host}:#{@port}"
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,35 @@
1
+ #
2
+ # Define the versions of the gem.
3
+ #
4
+ module DruidConfig
5
+ #
6
+ # Commmon functions for the gem
7
+ #
8
+ module Util
9
+ #
10
+ # This method is used to protect the Gem to API errors. If a query fails,
11
+ # the client will be reset and try the query to new coordinator. If it
12
+ # fails too, a DruidApiError will be launched.
13
+ #
14
+ # If the error comes from another point of the code, the Exception
15
+ # is launched normally
16
+ #
17
+ def secure_query
18
+ return unless block_given?
19
+ @retries = 0
20
+ begin
21
+ yield
22
+ rescue HTTParty::RedirectionTooDeep => e
23
+ raise(DruidApiError, e) if @retries > 0
24
+ @retries += 1
25
+ reset!
26
+ retry
27
+ rescue Errno::ECONNREFUSED => e
28
+ raise(DruidApiError, e) if @retries > 0
29
+ @retries += 1
30
+ reset!
31
+ retry
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,12 @@
1
+ #
2
+ # Define the versions of the gem.
3
+ #
4
+ module DruidConfig
5
+ module Version
6
+ # Version of the gem
7
+ VERSION = '0.1.0'
8
+
9
+ # Base URI foor coordinator queries
10
+ API_VERSION = 'v1'
11
+ end
12
+ end
@@ -0,0 +1,216 @@
1
+ require 'zk'
2
+ require 'rest_client'
3
+
4
+ module DruidConfig
5
+ #
6
+ # Class to connect and get information about nodes in cluster using
7
+ # Zookeeper
8
+ #
9
+ class ZK
10
+ # Coordinator service
11
+ COORDINATOR = 'coordinator'
12
+ OVERLORD = 'overlord'
13
+ SERVICES = [COORDINATOR, OVERLORD]
14
+
15
+ #
16
+ # Initialize variables and call register
17
+ #
18
+ # == Parameters:
19
+ # uri::
20
+ # Uri of zookeper
21
+ # opts::
22
+ # Hash with options:
23
+ # - discovery_path: Custom URL of discovery path for Druid
24
+ #
25
+ def initialize(uri, opts = {})
26
+ # Control Zookeper connection
27
+ @zk = ::ZK.new(uri, chroot: :check)
28
+ @registry = Hash.new { |hash, key| hash[key] = [] }
29
+ @discovery_path = opts[:discovery_path] || '/discovery'
30
+ @watched_services = {}
31
+ register
32
+ end
33
+
34
+ #
35
+ # Load the data from Zookeeper
36
+ #
37
+ def register
38
+ $log.info('druid.zk register discovery path') if $log
39
+ @zk.on_expired_session { register }
40
+ @zk.register(@discovery_path, only: :child) do
41
+ $log.info('druid.zk got event on discovery path') if $log
42
+ check_services
43
+ end
44
+ check_services
45
+ end
46
+
47
+ #
48
+ # Force to close Zookeper connection
49
+ #
50
+ def close!
51
+ $log.info('druid.zk shutting down') if $log
52
+ @zk.close!
53
+ end
54
+
55
+ #
56
+ # Return the URI of a random available coordinator.
57
+ # Poor mans load balancing
58
+ #
59
+ def coordinator
60
+ random_node(COORDINATOR)
61
+ end
62
+
63
+ #
64
+ # Return the URI of a random available overlord.
65
+ # Poor mans load balancing
66
+ #
67
+ def overlord
68
+ random_node(OVERLORD)
69
+ end
70
+
71
+ #
72
+ # Return a random value of a service
73
+ #
74
+ # == Parameters:
75
+ # service::
76
+ # String with the name of the service
77
+ #
78
+ def random_node(service)
79
+ return nil if @registry[service].size == 0
80
+ # Return a random broker from available brokers
81
+ i = Random.rand(@registry[service].size)
82
+ @registry[service][i][:uri]
83
+ end
84
+
85
+ #
86
+ # Register a new service
87
+ #
88
+ def register_service(service, brokers)
89
+ $log.info("druid.zk register", service: service, brokers: brokers) if $log
90
+ # poor mans load balancing
91
+ @registry[service] = brokers.shuffle
92
+ end
93
+
94
+ #
95
+ # Unregister a service
96
+ #
97
+ def unregister_service(service)
98
+ $log.info("druid.zk unregister", service: service) if $log
99
+ @registry.delete(service)
100
+ unwatch_service(service)
101
+ end
102
+
103
+ #
104
+ # Set a watcher for a service
105
+ #
106
+ def watch_service(service)
107
+ return if @watched_services.include?(service)
108
+ $log.info("druid.zk watch", service: service) if $log
109
+ watch = @zk.register(watch_path(service), only: :child) do |event|
110
+ $log.info("druid.zk got event on watch path for", service: service, event: event) if $log
111
+ unwatch_service(service)
112
+ check_service(service)
113
+ end
114
+ @watched_services[service] = watch
115
+ end
116
+
117
+ #
118
+ # Unset a service to watch
119
+ #
120
+ def unwatch_service(service)
121
+ return unless @watched_services.include?(service)
122
+ $log.info("druid.zk unwatch", service: service) if $log
123
+ @watched_services.delete(service).unregister
124
+ end
125
+
126
+ #
127
+ # Check current services
128
+ #
129
+ def check_services
130
+ $log.info("druid.zk checking services") if $log
131
+ zk_services = @zk.children(@discovery_path, watch: true)
132
+
133
+ (services - zk_services).each do |service|
134
+ unregister_service(service)
135
+ end
136
+
137
+ zk_services.each do |service|
138
+ check_service(service)
139
+ end
140
+ end
141
+
142
+ #
143
+ # Verify is a Coordinator is available
144
+ #
145
+ # == Parameters:
146
+ # name::
147
+ # String with the name of the coordinator
148
+ # service::
149
+ # String with the service
150
+ #
151
+ # == Returns:
152
+ # URI of the coordinator or false
153
+ #
154
+ def verify_node(name, service)
155
+ $log.info("druid.zk verify", node: name, service: service) if $log
156
+ info = @zk.get("#{watch_path(service)}/#{name}")
157
+ node = JSON.parse(info[0])
158
+ uri = "http://#{node['address']}:#{node['port']}/"
159
+ check = RestClient::Request.execute(
160
+ method: :get, url: "#{uri}status",
161
+ timeout: 5, open_timeout: 5
162
+ )
163
+ $log.info("druid.zk verified", uri: uri, sources: check) if $log
164
+ return uri if check.code == 200
165
+ rescue
166
+ return false
167
+ end
168
+
169
+ #
170
+ # Watch path of a service
171
+ #
172
+ def watch_path(service)
173
+ "#{@discovery_path}/#{service}"
174
+ end
175
+
176
+ #
177
+ # Check a service
178
+ #
179
+ def check_service(service)
180
+ return if @watched_services.include?(service) ||
181
+ !SERVICES.include?(service)
182
+
183
+ # Start to watch this service
184
+ watch_service(service)
185
+
186
+ known = @registry[service].map { |node| node[:name] }
187
+ live = @zk.children(watch_path(service), watch: true)
188
+ new_list = @registry[service].select { |node| live.include?(node[:name]) }
189
+ $log.info("druid.zk checking", service: service, known: known, live: live, new_list: new_list) if $log
190
+
191
+ # verify the new entries to be living brokers
192
+ (live - known).each do |name|
193
+ uri = verify_node(name, service)
194
+ new_list.push(name: name, uri: uri) if uri
195
+ end
196
+
197
+ if new_list.empty?
198
+ # don't show services w/o active brokers
199
+ unregister_service(service)
200
+ else
201
+ register_service(service, new_list)
202
+ end
203
+ end
204
+
205
+ #
206
+ # Get all available services
207
+ #
208
+ def services
209
+ @registry.keys
210
+ end
211
+
212
+ def to_s
213
+ @registry.to_s
214
+ end
215
+ end
216
+ end
@@ -0,0 +1,39 @@
1
+ # Global library
2
+ require 'httparty'
3
+
4
+ # Classes
5
+ require 'druid_config/zk'
6
+ require 'druid_config/version'
7
+ require 'druid_config/util'
8
+ require 'druid_config/entities/segment'
9
+ require 'druid_config/entities/worker'
10
+ require 'druid_config/entities/node'
11
+ require 'druid_config/entities/tier'
12
+ require 'druid_config/entities/data_source'
13
+ require 'druid_config/cluster'
14
+ require 'druid_config/client'
15
+
16
+ # Base namespace of the gem
17
+ module DruidConfig
18
+ #
19
+ # Exception class for an error to connect the API
20
+ #
21
+ class DruidApiError < StandardError; end
22
+
23
+ # Global client of Druidconfig module
24
+ @client = nil
25
+
26
+ #
27
+ # Initialize the current client
28
+ #
29
+ def self.client=(client)
30
+ @client = client
31
+ end
32
+
33
+ #
34
+ # Return initialized client
35
+ #
36
+ def self.client
37
+ @client
38
+ end
39
+ end
@@ -0,0 +1,32 @@
1
+ require 'spec_helper'
2
+ require 'pry'
3
+ require 'pry-nav'
4
+
5
+ # describe DruidConfig::Cluster do
6
+ # before(:each) do
7
+ # @cluster = DruidConfig::Cluster.new('localhost', zk_keepalive: true)
8
+ # end
9
+
10
+ # it 'must get the leader' do
11
+ # expect(@cluster.leader).to eq 'coordinator.stub'
12
+ # end
13
+
14
+ # it 'must get load datasources of a cluster' do
15
+ # datasources = @cluster.datasources
16
+
17
+
18
+ # # basic = @cluster.load_status
19
+ # # expect(basic.keys).to eq %w(datasource1 datasource2)
20
+ # # expect(basic[basic.keys.first]).to eq 100
21
+
22
+ # # simple = @cluster.load_status('simple')
23
+ # # expect(simple.keys).to eq %w(datasource1 datasource2)
24
+ # # expect(simple[simple.keys.first]).to eq 0
25
+
26
+ # # # Use tiers
27
+ # # simple = @cluster.load_status('full')
28
+ # # expect(simple.keys).to eq %w(_default_tier hot)
29
+ # # expect(simple['_default_tier']['datasource1']).to eq 0
30
+ # # expect(simple['hot']['datasource2']).to eq 0
31
+ # end
32
+ # end
@@ -0,0 +1,17 @@
1
+ require 'spec_helper'
2
+
3
+ describe DruidConfig::Entities::DataSource do
4
+ before(:each) do
5
+ @name = 'datasource'
6
+ @properties = { 'client' => 'side' }
7
+ @metadata = { 'name' => @name, 'properties' => @properties }
8
+ @load_status = 100
9
+ end
10
+
11
+ it 'initialize the model based on metadata' do
12
+ datasource = DruidConfig::Entities::DataSource.new(@metadata, @load_status)
13
+ expect(datasource.name).to eq @name
14
+ expect(datasource.properties).to eq @properties
15
+ expect(datasource.load_status).to eq @load_status
16
+ end
17
+ end
data/spec/node_spec.rb ADDED
@@ -0,0 +1,63 @@
1
+ require 'spec_helper'
2
+
3
+ describe DruidConfig::Entities::Node do
4
+ before(:each) do
5
+ @host = 'stubbed.cluster:8083'
6
+ @max_size = 100_000
7
+ @type = 'historical'
8
+ @priority = 0
9
+ @segments = {
10
+ 'datasource_2015-10-22T15:00:00.000Z_2015-10-22T16:00:00.000Z_2015-10-22T15:00:17.214Z' => {
11
+ 'dataSource' => 'datasource',
12
+ 'interval' => '2015-10-22T15:00:00.000Z/2015-10-22T16:00:00.000Z',
13
+ 'version' => '2015-10-22T15:00:17.214Z',
14
+ 'loadSpec' => {},
15
+ 'dimensions' => '',
16
+ 'metrics' => 'events,sum_bytes',
17
+ 'shardSpec' => {
18
+ 'type' => 'linear',
19
+ 'partitionNum' => 0
20
+ },
21
+ 'binaryVersion' => nil,
22
+ 'size' => 0,
23
+ 'identifier' => 'datasource_2015-10-22T15:00:00.000Z_2015-10-22T16:00:00.000Z_2015-10-22T15:00:17.214Z'
24
+ }
25
+ }
26
+ @size = 50_000
27
+ @metadata = { 'host' => @host, 'maxSize' => @max_size, 'type' => @type,
28
+ 'priority' => 0, 'segments' => @segments, 'currSize' => @size }
29
+
30
+ @queue = {
31
+ 'segmentsToLoad' => [],
32
+ 'segmentsToDrop' => []
33
+ }
34
+ end
35
+
36
+ it 'initialize a Node based on metadata' do
37
+ datasource = DruidConfig::Entities::Node.new(@metadata, @queue)
38
+ expect(datasource.host).to eq @host.split(':').first
39
+ expect(datasource.uri).to eq @host
40
+ expect(datasource.max_size).to eq @max_size
41
+ expect(datasource.type).to eq @type.to_sym
42
+ expect(datasource.priority).to eq @priority
43
+ expect(datasource.size).to eq @size
44
+ expect(datasource.segments_to_load).to eq []
45
+ expect(datasource.segments_to_drop).to eq []
46
+ end
47
+
48
+ it 'calculate free space' do
49
+ datasource = DruidConfig::Entities::Node.new(@metadata, @queue)
50
+ expect(datasource.free).to eq(@max_size - @size)
51
+ end
52
+
53
+ it 'calculate percentage of used space' do
54
+ datasource = DruidConfig::Entities::Node.new(@metadata, @queue)
55
+ expect(datasource.used_percent).to eq((@size.to_f / @max_size) * 100)
56
+ end
57
+
58
+ it 'return 0 when max size is 0' do
59
+ datasource =
60
+ DruidConfig::Entities::Node.new(@metadata.merge('maxSize' => 0), @queue)
61
+ expect(datasource.used_percent).to eq 0
62
+ end
63
+ end
@@ -0,0 +1,79 @@
1
+ require 'druid_config'
2
+ require 'webmock/rspec'
3
+
4
+ # Mock Druid
5
+ ENV['MOCK_DRUID'] ||= 'false'
6
+
7
+ if ENV['MOCK_DRUID'] == 'true'
8
+ # Disable external connections
9
+ WebMock.disable_net_connect!(allow_localhost: true)
10
+ end
11
+
12
+ RSpec.configure do |config|
13
+ config.expect_with :rspec do |expectations|
14
+ # This option will default to `true` in RSpec 4. It makes the `description`
15
+ # and `failure_message` of custom matchers include text for helper methods
16
+ # defined using `chain`, e.g.:
17
+ # be_bigger_than(2).and_smaller_than(4).description
18
+ # # => "be bigger than 2 and smaller than 4"
19
+ # ...rather than:
20
+ # # => "be bigger than 2"
21
+ expectations.include_chain_clauses_in_custom_matcher_descriptions = true
22
+ end
23
+
24
+ # Use color in STDOUT
25
+ config.color = true
26
+ # Use color not only in STDOUT but also in pagers and files
27
+ config.tty = true
28
+ # Use the specified formatter
29
+ config.formatter = :documentation # :progress, :html, :textmate
30
+
31
+ config.mock_with :rspec do |mocks|
32
+ # Prevents you from mocking or stubbing a method that does not exist on
33
+ # a real object. This is generally recommended, and will default to
34
+ # `true` in RSpec 4.
35
+ mocks.verify_partial_doubles = true
36
+ end
37
+
38
+ #
39
+ # Mock druid API queries
40
+ #
41
+ config.before(:each) do
42
+ if ENV['MOCK_DRUID'] == 'true'
43
+ # Stub DruidConfig::Client to ignore Zookeeper.
44
+ # TODO: We must improve it!!!
45
+ class ClientStub
46
+ def coordinator
47
+ 'coordinator.stub/'
48
+ end
49
+ end
50
+ allow(DruidConfig).to receive(:client) { ClientStub.new }
51
+
52
+ # Stub queries
53
+ # ----------------------------------
54
+
55
+ # Our scenario:
56
+ # leader: coordinator.stub
57
+ # datasources: datasource1, datasource2
58
+ # tiers: _default_tier, hot
59
+ # stub_request(:get, 'http://coordinator.stub/druid/coordinator/v1/leader')
60
+ # .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' })
61
+ # .to_return(status: 200, body: 'coordinator.stub', headers: {})
62
+
63
+ # stub_request(:get, 'http://coordinator.stub/druid/coordinator/v1/loadstatus')
64
+ # .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' })
65
+ # .to_return(status: 200, body: '{"datasource1":100.0,"datasource2":100.0}',
66
+ # headers: { 'Content-Type' => 'application/json' })
67
+
68
+ # stub_request(:get, 'http://coordinator.stub/druid/coordinator/v1/loadstatus?simple')
69
+ # .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' })
70
+ # .to_return(status: 200, body: '{"datasource1":0,"datasource2":0}',
71
+ # headers: { 'Content-Type' => 'application/json' })
72
+
73
+ # stub_request(:get, 'http://coordinator.stub/druid/coordinator/v1/loadstatus?full')
74
+ # .with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' })
75
+ # .to_return(status: 200, body: '{"_default_tier":{"datasource1":0}, "hot":{"datasource2":0}}',
76
+ # headers: { 'Content-Type' => 'application/json' })
77
+ end
78
+ end
79
+ end