druid_config 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +12 -6
- data/lib/druid_config.rb +1 -0
- data/lib/druid_config/cluster.rb +66 -30
- data/lib/druid_config/entities/data_source.rb +26 -13
- data/lib/druid_config/entities/node.rb +65 -13
- data/lib/druid_config/entities/task.rb +97 -0
- data/lib/druid_config/entities/tier.rb +18 -4
- data/lib/druid_config/entities/worker.rb +12 -4
- data/lib/druid_config/util.rb +34 -0
- data/lib/druid_config/version.rb +1 -1
- data/lib/druid_config/zk.rb +6 -0
- data/spec/cluster_spec.rb +20 -24
- data/spec/data/druid_responses.yml +4 -0
- data/spec/node_spec.rb +6 -4
- data/spec/spec_helper.rb +22 -16
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a03eb8b4f7df4223fe098516a23443beac52be00
|
4
|
+
data.tar.gz: 230450c3d105e2327c681f1bc1beb225926361be
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 151efb283126cbf585549a0e1d0a7f766f48c51e3f21e18ab373c34d59a247c64fbcc1188157ccaadb7b5940ccdcc480c24b87f33a2f0e890e019be817b9bcab
|
7
|
+
data.tar.gz: db35a7cc044f47bb4de83ea1467e35186389471485f71f48e3f4363087dc8f9326150a0df93a7844d901b804bf38348d51ab2cfb452a2aa708240cbd08a5ba76
|
data/README.md
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
[](https://badge.fury.io/rb/druid_config) [](https://travis-ci.org/redBorder/druid_config)
|
2
|
+
|
1
3
|
# DruidConfig
|
2
4
|
|
3
5
|
DruidConfig is a gem to access the information about Druid cluster status. You can check a node capacity, number of segments, tiers... It uses [zookeeper](https://zookeeper.apache.org/) to get coordinator and overlord URIs.
|
@@ -5,9 +7,13 @@ DruidConfig is a gem to access the information about Druid cluster status. You c
|
|
5
7
|
To use in your application, add this line to your Gemfile:
|
6
8
|
|
7
9
|
```ruby
|
8
|
-
|
10
|
+
gem 'druid_config'
|
9
11
|
```
|
10
12
|
|
13
|
+
## Query Druid data
|
14
|
+
|
15
|
+
The purpose of this gem is to query Druid cluster status. If you want to query Druid data, I recommend you to use [ruby-druid gem](https://github.com/ruby-druid/ruby-druid).
|
16
|
+
|
11
17
|
# Initialization
|
12
18
|
|
13
19
|
`Cluster` is the base class to perform queries. To initialize it send the Zookeeper URI and options as arguments:
|
@@ -44,11 +50,11 @@ Call methods defined in `DruidConfig::Cluster` to access to the data. To get mor
|
|
44
50
|
|
45
51
|
Some methods return an instance of an `Entity` class. These entities provide multiple methods to access data. Defined entities are inside `druid_config/entities` folder.
|
46
52
|
|
47
|
-
* [DataSource]()
|
48
|
-
* [
|
49
|
-
* [
|
50
|
-
* [
|
51
|
-
* [Worker]()
|
53
|
+
* [DataSource](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/data_source.rb)
|
54
|
+
* [Segment](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/segment.rb)
|
55
|
+
* [Tier](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/tier.rb)
|
56
|
+
* [Node](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/node.rb)
|
57
|
+
* [Worker](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/worker.rb)
|
52
58
|
|
53
59
|
## Exceptions
|
54
60
|
|
data/lib/druid_config.rb
CHANGED
@@ -6,6 +6,7 @@ require 'druid_config/zk'
|
|
6
6
|
require 'druid_config/version'
|
7
7
|
require 'druid_config/util'
|
8
8
|
require 'druid_config/entities/segment'
|
9
|
+
require 'druid_config/entities/task'
|
9
10
|
require 'druid_config/entities/worker'
|
10
11
|
require 'druid_config/entities/node'
|
11
12
|
require 'druid_config/entities/tier'
|
data/lib/druid_config/cluster.rb
CHANGED
@@ -136,7 +136,7 @@ module DruidConfig
|
|
136
136
|
def datasources
|
137
137
|
datasource_status = load_status
|
138
138
|
secure_query do
|
139
|
-
self.class.get('/datasources?
|
139
|
+
self.class.get('/datasources?simple').map do |data|
|
140
140
|
DruidConfig::Entities::DataSource.new(
|
141
141
|
data,
|
142
142
|
datasource_status.select { |k, _| k == data['name'] }.values.first)
|
@@ -202,8 +202,8 @@ module DruidConfig
|
|
202
202
|
#
|
203
203
|
def servers
|
204
204
|
secure_query do
|
205
|
-
queue = load_queue('
|
206
|
-
self.class.get('/servers?
|
205
|
+
queue = load_queue('simple')
|
206
|
+
self.class.get('/servers?simple').map do |data|
|
207
207
|
DruidConfig::Entities::Node.new(
|
208
208
|
data,
|
209
209
|
queue.select { |k, _| k == data['host'] }.values.first)
|
@@ -253,22 +253,14 @@ module DruidConfig
|
|
253
253
|
# Array of Workers
|
254
254
|
#
|
255
255
|
def workers
|
256
|
-
# Stash the base_uri
|
257
|
-
stash_uri
|
258
|
-
self.class.base_uri(
|
259
|
-
"#{DruidConfig.client.overlord}"\
|
260
|
-
"druid/indexer/#{DruidConfig::Version::API_VERSION}")
|
261
256
|
workers = []
|
262
257
|
# Perform a query
|
263
|
-
|
258
|
+
query_overlord do
|
264
259
|
secure_query do
|
265
260
|
workers = self.class.get('/workers').map do |worker|
|
266
261
|
DruidConfig::Entities::Worker.new(worker)
|
267
262
|
end
|
268
263
|
end
|
269
|
-
ensure
|
270
|
-
# Recover it
|
271
|
-
pop_uri
|
272
264
|
end
|
273
265
|
# Return
|
274
266
|
workers
|
@@ -281,6 +273,68 @@ module DruidConfig
|
|
281
273
|
@physical_workers ||= workers.map(&:host).uniq
|
282
274
|
end
|
283
275
|
|
276
|
+
# Tasks
|
277
|
+
# -----------------
|
278
|
+
|
279
|
+
#
|
280
|
+
# Return tasks based on the status
|
281
|
+
#
|
282
|
+
# == Returns:
|
283
|
+
# Array of Tasks
|
284
|
+
#
|
285
|
+
%w(running pending waiting).each do |status|
|
286
|
+
define_method("#{status}_tasks") do
|
287
|
+
tasks = []
|
288
|
+
query_overlord do
|
289
|
+
tasks = self.class.get("/#{status}Tasks").map do |task|
|
290
|
+
DruidConfig::Entities::Task.new(
|
291
|
+
task['id'],
|
292
|
+
DruidConfig::Entities::Task::STATUS[status.to_sym],
|
293
|
+
created_time: task['createdTime'],
|
294
|
+
query_insertion_time: task['queueInsertionTime'])
|
295
|
+
end
|
296
|
+
end
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
300
|
+
#
|
301
|
+
# Find a task
|
302
|
+
#
|
303
|
+
def task(id)
|
304
|
+
DruidConfig::Entities::Task.find(id)
|
305
|
+
end
|
306
|
+
|
307
|
+
#
|
308
|
+
# Return complete tasks
|
309
|
+
#
|
310
|
+
# == Returns:
|
311
|
+
# Array of Tasks
|
312
|
+
#
|
313
|
+
def complete_tasks
|
314
|
+
tasks = []
|
315
|
+
query_overlord do
|
316
|
+
tasks = self.class.get('/completeTasks').map do |task|
|
317
|
+
DruidConfig::Entities::Task.new(
|
318
|
+
task['id'],
|
319
|
+
task['statusCode'])
|
320
|
+
end
|
321
|
+
end
|
322
|
+
end
|
323
|
+
|
324
|
+
#
|
325
|
+
# Return failed completed tasks
|
326
|
+
#
|
327
|
+
def failed_tasks
|
328
|
+
complete_tasks.select(&:failed?)
|
329
|
+
end
|
330
|
+
|
331
|
+
#
|
332
|
+
# Return success completed tasks
|
333
|
+
#
|
334
|
+
def success_tasks
|
335
|
+
complete_tasks.select(&:success?)
|
336
|
+
end
|
337
|
+
|
284
338
|
# Services
|
285
339
|
# -----------------
|
286
340
|
|
@@ -302,23 +356,5 @@ module DruidConfig
|
|
302
356
|
# Return nodes
|
303
357
|
@services = services
|
304
358
|
end
|
305
|
-
|
306
|
-
private
|
307
|
-
|
308
|
-
#
|
309
|
-
# Stash current base_uri
|
310
|
-
#
|
311
|
-
def stash_uri
|
312
|
-
@uri_stack ||= []
|
313
|
-
@uri_stack.push self.class.base_uri
|
314
|
-
end
|
315
|
-
|
316
|
-
#
|
317
|
-
# Pop next base_uri
|
318
|
-
#
|
319
|
-
def pop_uri
|
320
|
-
return if @uri_stack.nil? || @uri_stack.empty?
|
321
|
-
self.class.base_uri(@uri_stack.pop)
|
322
|
-
end
|
323
359
|
end
|
324
360
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
module DruidConfig
|
2
2
|
#
|
3
|
-
# Module of
|
3
|
+
# Module of Entities
|
4
4
|
#
|
5
5
|
module Entities
|
6
6
|
#
|
@@ -8,6 +8,7 @@ module DruidConfig
|
|
8
8
|
class DataSource
|
9
9
|
# HTTParty Rocks!
|
10
10
|
include HTTParty
|
11
|
+
include DruidConfig::Util
|
11
12
|
|
12
13
|
attr_reader :name, :properties, :load_status
|
13
14
|
|
@@ -31,28 +32,36 @@ module DruidConfig
|
|
31
32
|
# http://druid.io/docs/0.8.1/design/coordinator.html
|
32
33
|
#
|
33
34
|
|
34
|
-
def info
|
35
|
-
|
35
|
+
def info
|
36
|
+
secure_query do
|
37
|
+
@info ||= self.class.get("/datasources/#{@name}")
|
38
|
+
end
|
36
39
|
end
|
37
40
|
|
38
41
|
# Intervals
|
39
42
|
# -----------------
|
40
43
|
def intervals(params = '')
|
41
|
-
|
44
|
+
secure_query do
|
45
|
+
self.class.get("/datasources/#{@name}/intervals?#{params}")
|
46
|
+
end
|
42
47
|
end
|
43
48
|
|
44
49
|
def interval(interval, params = '')
|
45
|
-
|
46
|
-
|
50
|
+
secure_query do
|
51
|
+
self.class.get("/datasources/#{@name}/intervals/#{interval}"\
|
52
|
+
"?#{params}")
|
53
|
+
end
|
47
54
|
end
|
48
55
|
|
49
56
|
# Segments and Tiers
|
50
57
|
# -----------------
|
51
58
|
def segments
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
59
|
+
secure_query do
|
60
|
+
@segments ||=
|
61
|
+
self.class.get("/datasources/#{@name}/segments?full").map do |s|
|
62
|
+
DruidConfig::Entities::Segment.new(s)
|
63
|
+
end
|
64
|
+
end
|
56
65
|
end
|
57
66
|
|
58
67
|
def segment(segment)
|
@@ -66,12 +75,16 @@ module DruidConfig
|
|
66
75
|
# Rules
|
67
76
|
# -----------------
|
68
77
|
def rules(params = '')
|
69
|
-
|
78
|
+
secure_query do
|
79
|
+
self.class.get("/rules/#{@name}?#{params}")
|
80
|
+
end
|
70
81
|
end
|
71
82
|
|
72
83
|
def history_rules(interval)
|
73
|
-
|
74
|
-
|
84
|
+
secure_query do
|
85
|
+
self.class.get("/rules/#{@name}/history"\
|
86
|
+
"?interval=#{interval}")
|
87
|
+
end
|
75
88
|
end
|
76
89
|
end
|
77
90
|
end
|
@@ -6,10 +6,11 @@ module DruidConfig
|
|
6
6
|
class Node
|
7
7
|
# HTTParty Rocks!
|
8
8
|
include HTTParty
|
9
|
+
include DruidConfig::Util
|
9
10
|
|
10
11
|
# Readers
|
11
12
|
attr_reader :host, :port, :max_size, :type, :tier, :priority, :size,
|
12
|
-
:
|
13
|
+
:segments_to_load_count, :segments_to_drop_count,
|
13
14
|
:segments_to_load_size, :segments_to_drop_size
|
14
15
|
|
15
16
|
#
|
@@ -28,21 +29,20 @@ module DruidConfig
|
|
28
29
|
@tier = metadata['tier']
|
29
30
|
@priority = metadata['priority']
|
30
31
|
@size = metadata['currSize']
|
31
|
-
|
32
|
-
|
33
|
-
|
32
|
+
# Set end point for HTTParty
|
33
|
+
self.class.base_uri(
|
34
|
+
"#{DruidConfig.client.coordinator}"\
|
35
|
+
"druid/coordinator/#{DruidConfig::Version::API_VERSION}")
|
36
|
+
|
37
|
+
# Load more data from queue
|
34
38
|
if queue.nil?
|
35
|
-
@
|
39
|
+
@segments_to_load_count, @segments_to_drop_count = 0, 0
|
36
40
|
@segments_to_load_size, @segments_to_drop_size = 0, 0
|
37
41
|
else
|
38
|
-
@
|
39
|
-
|
40
|
-
|
41
|
-
@
|
42
|
-
DruidConfig::Entities::Segment.new(segment)
|
43
|
-
end
|
44
|
-
@segments_to_load_size = @segments_to_load.map(&:size).reduce(:+)
|
45
|
-
@segments_to_drop_size = @segments_to_drop.map(&:size).reduce(:+)
|
42
|
+
@segments_to_load_count = queue['segmentsToLoad']
|
43
|
+
@segments_to_drop_count = queue['segmentsToDrop']
|
44
|
+
@segments_to_load_size = queue['segmentsToLoadSize']
|
45
|
+
@segments_to_drop_size = queue['segmentsToLoadSize']
|
46
46
|
end
|
47
47
|
end
|
48
48
|
|
@@ -63,12 +63,64 @@ module DruidConfig
|
|
63
63
|
max_size - size
|
64
64
|
end
|
65
65
|
|
66
|
+
#
|
67
|
+
# Return all segments of this node
|
68
|
+
#
|
69
|
+
def segments_count
|
70
|
+
@segments_count ||=
|
71
|
+
self.class.get("/servers/#{uri}/segments").size
|
72
|
+
end
|
73
|
+
|
74
|
+
#
|
75
|
+
# Return all segments of this node
|
76
|
+
#
|
77
|
+
def segments
|
78
|
+
@segments ||=
|
79
|
+
self.class.get("/servers/#{uri}/segments?full").map do |s|
|
80
|
+
DruidConfig::Entities::Segment.new(s)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
#
|
85
|
+
# Get segments to load
|
86
|
+
#
|
87
|
+
def segments_to_load
|
88
|
+
current_queue = queue
|
89
|
+
return [] unless current_queue
|
90
|
+
current_queue['segmentsToLoad'].map do |segment|
|
91
|
+
DruidConfig::Entities::Segment.new(segment)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
#
|
96
|
+
# Get segments to drop
|
97
|
+
#
|
98
|
+
def segments_to_drop
|
99
|
+
current_queue = queue
|
100
|
+
return [] unless current_queue
|
101
|
+
current_queue['segmentsToDrop'].map do |segment|
|
102
|
+
DruidConfig::Entities::Segment.new(segment)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
66
106
|
#
|
67
107
|
# Return the URI of this node
|
68
108
|
#
|
69
109
|
def uri
|
70
110
|
"#{@host}:#{@port}"
|
71
111
|
end
|
112
|
+
|
113
|
+
private
|
114
|
+
|
115
|
+
#
|
116
|
+
# Get load queue for this node
|
117
|
+
#
|
118
|
+
def queue
|
119
|
+
secure_query do
|
120
|
+
self.class.get('/loadqueue?full')
|
121
|
+
.select { |k, _| k == uri }.values.first
|
122
|
+
end
|
123
|
+
end
|
72
124
|
end
|
73
125
|
end
|
74
126
|
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
module DruidConfig
|
2
|
+
#
|
3
|
+
# Module of Entities
|
4
|
+
#
|
5
|
+
module Entities
|
6
|
+
#
|
7
|
+
# Init a DataSource
|
8
|
+
class Task
|
9
|
+
# HTTParty Rocks!
|
10
|
+
include HTTParty
|
11
|
+
include DruidConfig::Util
|
12
|
+
|
13
|
+
# Statuses constants
|
14
|
+
STATUS = {
|
15
|
+
running: 'RUNNING',
|
16
|
+
pending: 'PENDING',
|
17
|
+
success: 'SUCCESS',
|
18
|
+
waiting: 'WAITING',
|
19
|
+
failed: 'FAILED'
|
20
|
+
}
|
21
|
+
|
22
|
+
attr_reader :id, :status, :created_time, :query_insertion_time
|
23
|
+
|
24
|
+
#
|
25
|
+
# Find a task based on id
|
26
|
+
#
|
27
|
+
def self.find(id)
|
28
|
+
# Set end point for HTTParty
|
29
|
+
base_uri(
|
30
|
+
"#{DruidConfig.client.overlord}"\
|
31
|
+
"druid/indexer/#{DruidConfig::Version::API_VERSION}/task")
|
32
|
+
# Get data
|
33
|
+
status = get("/#{id}/status")
|
34
|
+
new(id, status['status']['status'])
|
35
|
+
end
|
36
|
+
|
37
|
+
#
|
38
|
+
# Initialize a task
|
39
|
+
#
|
40
|
+
# == Parameters:
|
41
|
+
# id::
|
42
|
+
# String with identifier
|
43
|
+
# status::
|
44
|
+
# Current status of the task
|
45
|
+
#
|
46
|
+
def initialize(id, status, extended_info = {})
|
47
|
+
@id = id
|
48
|
+
@status = status.downcase.to_sym
|
49
|
+
@created_time = extended_info[:created_time]
|
50
|
+
@query_insertion_time = extended_info[:query_insertion_time]
|
51
|
+
# Set end point for HTTParty
|
52
|
+
self.class.base_uri(
|
53
|
+
"#{DruidConfig.client.overlord}"\
|
54
|
+
"druid/indexer/#{DruidConfig::Version::API_VERSION}/task")
|
55
|
+
end
|
56
|
+
|
57
|
+
#
|
58
|
+
# Multiple methods to check status of the tasks
|
59
|
+
#
|
60
|
+
STATUS.keys.each do |s|
|
61
|
+
define_method("#{s}?") do
|
62
|
+
s.to_sym == @status
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
alias_method :completed?, :success?
|
67
|
+
|
68
|
+
#
|
69
|
+
# Get payload of the task
|
70
|
+
#
|
71
|
+
def payload
|
72
|
+
@payload ||= self.class.get("/#{@id}")['payload']
|
73
|
+
end
|
74
|
+
|
75
|
+
#
|
76
|
+
# Current log of a task
|
77
|
+
#
|
78
|
+
def log
|
79
|
+
self.class.get("/#{@id}/log")
|
80
|
+
end
|
81
|
+
|
82
|
+
#
|
83
|
+
# Get the dataSource of this task
|
84
|
+
#
|
85
|
+
def datasource
|
86
|
+
payload['dataSource']
|
87
|
+
end
|
88
|
+
|
89
|
+
#
|
90
|
+
# Group of the task
|
91
|
+
#
|
92
|
+
def group_id
|
93
|
+
payload['groupId']
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -42,14 +42,28 @@ module DruidConfig
|
|
42
42
|
.flatten.sort_by { |seg| seg.interval.first }
|
43
43
|
end
|
44
44
|
|
45
|
+
def segments_count
|
46
|
+
@segments_count ||= nodes.map(&:segments_count).inject(:+)
|
47
|
+
end
|
48
|
+
|
45
49
|
def segments_to_load
|
46
|
-
@segments_to_load ||=
|
47
|
-
|
50
|
+
@segments_to_load ||= nodes.map(&:segments_to_load)
|
51
|
+
.flatten.sort_by { |seg| seg.interval.first }
|
48
52
|
end
|
49
53
|
|
50
54
|
def segments_to_drop
|
51
|
-
@segments_to_drop ||=
|
52
|
-
|
55
|
+
@segments_to_drop ||= nodes.map(&:segments_to_drop)
|
56
|
+
.flatten.sort_by { |seg| seg.interval.first }
|
57
|
+
end
|
58
|
+
|
59
|
+
def segments_to_load_count
|
60
|
+
@segments_to_load_count ||=
|
61
|
+
nodes.map(&:segments_to_load_count).inject(:+)
|
62
|
+
end
|
63
|
+
|
64
|
+
def segments_to_drop_count
|
65
|
+
@segments_to_drop_count ||=
|
66
|
+
nodes.map(&:segments_to_drop_count).inject(:+)
|
53
67
|
end
|
54
68
|
|
55
69
|
def segments_to_load_size
|
@@ -6,7 +6,7 @@ module DruidConfig
|
|
6
6
|
class Worker
|
7
7
|
# Readers
|
8
8
|
attr_reader :last_completed_task_time, :host, :port, :ip, :capacity,
|
9
|
-
|
9
|
+
:version, :running_tasks, :capacity_used
|
10
10
|
|
11
11
|
#
|
12
12
|
# Initialize it with received info
|
@@ -21,21 +21,29 @@ module DruidConfig
|
|
21
21
|
@capacity = metadata['worker']['capacity']
|
22
22
|
@version = metadata['worker']['version']
|
23
23
|
@last_completed_task_time = metadata['lastCompletedTaskTime']
|
24
|
-
@running_tasks = metadata['runningTasks']
|
24
|
+
@running_tasks = metadata['runningTasks'].map do |task|
|
25
|
+
DruidConfig::Entities::Task.new(
|
26
|
+
task,
|
27
|
+
DruidConfig::Entities::Task::STATUS[:running],
|
28
|
+
created_time: task['createdTime'],
|
29
|
+
query_insertion_time: task['queueInsertionTime'])
|
30
|
+
end
|
25
31
|
@capacity_used = metadata['currCapacityUsed']
|
26
32
|
end
|
27
33
|
|
34
|
+
alias_method :used, :capacity_used
|
35
|
+
|
28
36
|
#
|
29
37
|
# Return free capacity
|
30
38
|
#
|
31
39
|
def free
|
32
|
-
@free ||= (capacity -
|
40
|
+
@free ||= (capacity - capacity_used)
|
33
41
|
end
|
34
42
|
|
35
43
|
#
|
36
44
|
# Return capacity used
|
37
45
|
#
|
38
|
-
def
|
46
|
+
def used_percent
|
39
47
|
return 0 unless @capacity && @capacity != 0
|
40
48
|
((@capacity_used.to_f / @capacity) * 100).round(2)
|
41
49
|
end
|
data/lib/druid_config/util.rb
CHANGED
@@ -31,5 +31,39 @@ module DruidConfig
|
|
31
31
|
retry
|
32
32
|
end
|
33
33
|
end
|
34
|
+
|
35
|
+
#
|
36
|
+
# Update the URI of HTTParty to perform queries to Overlord.
|
37
|
+
# After perform the query, the URI is reverted to coordinator.
|
38
|
+
#
|
39
|
+
def query_overlord
|
40
|
+
return unless block_given?
|
41
|
+
stash_uri
|
42
|
+
self.class.base_uri(
|
43
|
+
"#{DruidConfig.client.overlord}"\
|
44
|
+
"druid/indexer/#{DruidConfig::Version::API_VERSION}")
|
45
|
+
begin
|
46
|
+
yield
|
47
|
+
ensure
|
48
|
+
# Ensure we revert the URI
|
49
|
+
pop_uri
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
#
|
54
|
+
# Stash current base_uri
|
55
|
+
#
|
56
|
+
def stash_uri
|
57
|
+
@uri_stack ||= []
|
58
|
+
@uri_stack.push self.class.base_uri
|
59
|
+
end
|
60
|
+
|
61
|
+
#
|
62
|
+
# Pop next base_uri
|
63
|
+
#
|
64
|
+
def pop_uri
|
65
|
+
return if @uri_stack.nil? || @uri_stack.empty?
|
66
|
+
self.class.base_uri(@uri_stack.pop)
|
67
|
+
end
|
34
68
|
end
|
35
69
|
end
|
data/lib/druid_config/version.rb
CHANGED
data/lib/druid_config/zk.rb
CHANGED
data/spec/cluster_spec.rb
CHANGED
@@ -2,31 +2,27 @@ require 'spec_helper'
|
|
2
2
|
require 'pry'
|
3
3
|
require 'pry-nav'
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
5
|
+
describe DruidConfig::Cluster do
|
6
|
+
before(:each) do
|
7
|
+
@cluster = DruidConfig::Cluster.new('localhost', zk_keepalive: true)
|
8
|
+
end
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
10
|
+
it 'must get the leader' do
|
11
|
+
expect(@cluster.leader).to eq 'coordinator.stub'
|
12
|
+
end
|
13
13
|
|
14
|
-
|
15
|
-
|
14
|
+
it 'must get load status' do
|
15
|
+
basic = @cluster.load_status
|
16
|
+
expect(basic.keys).to eq %w(datasource1 datasource2)
|
17
|
+
expect(basic[basic.keys.first]).to eq 100
|
16
18
|
|
19
|
+
simple = @cluster.load_status('simple')
|
20
|
+
expect(simple.keys).to eq %w(datasource1 datasource2)
|
21
|
+
expect(simple[simple.keys.first]).to eq 0
|
17
22
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
# # expect(simple[simple.keys.first]).to eq 0
|
25
|
-
|
26
|
-
# # # Use tiers
|
27
|
-
# # simple = @cluster.load_status('full')
|
28
|
-
# # expect(simple.keys).to eq %w(_default_tier hot)
|
29
|
-
# # expect(simple['_default_tier']['datasource1']).to eq 0
|
30
|
-
# # expect(simple['hot']['datasource2']).to eq 0
|
31
|
-
# end
|
32
|
-
# end
|
23
|
+
full = @cluster.load_status('full')
|
24
|
+
expect(full.keys).to eq %w(_default_tier hot)
|
25
|
+
expect(full['_default_tier']['datasource1']).to eq 0
|
26
|
+
expect(full['hot']['datasource2']).to eq 0
|
27
|
+
end
|
28
|
+
end
|
data/spec/node_spec.rb
CHANGED
@@ -28,8 +28,10 @@ describe DruidConfig::Entities::Node do
|
|
28
28
|
'priority' => 0, 'segments' => @segments, 'currSize' => @size }
|
29
29
|
|
30
30
|
@queue = {
|
31
|
-
'segmentsToLoad' =>
|
32
|
-
'segmentsToDrop' =>
|
31
|
+
'segmentsToLoad' => 0,
|
32
|
+
'segmentsToDrop' => 0,
|
33
|
+
'segmentsToLoadSize' => 0,
|
34
|
+
'segmentsToDropSize' => 0
|
33
35
|
}
|
34
36
|
end
|
35
37
|
|
@@ -41,8 +43,8 @@ describe DruidConfig::Entities::Node do
|
|
41
43
|
expect(datasource.type).to eq @type.to_sym
|
42
44
|
expect(datasource.priority).to eq @priority
|
43
45
|
expect(datasource.size).to eq @size
|
44
|
-
expect(datasource.
|
45
|
-
expect(datasource.
|
46
|
+
expect(datasource.segments_to_load_count).to eq 0
|
47
|
+
expect(datasource.segments_to_drop_count).to eq 0
|
46
48
|
end
|
47
49
|
|
48
50
|
it 'calculate free space' do
|
data/spec/spec_helper.rb
CHANGED
@@ -46,8 +46,11 @@ RSpec.configure do |config|
|
|
46
46
|
def coordinator
|
47
47
|
'coordinator.stub/'
|
48
48
|
end
|
49
|
+
def overlord
|
50
|
+
'overlord.stub/'
|
51
|
+
end
|
49
52
|
end
|
50
|
-
allow(DruidConfig).to receive(:
|
53
|
+
allow(DruidConfig::Client).to receive(:new) { ClientStub.new }
|
51
54
|
|
52
55
|
# Stub queries
|
53
56
|
# ----------------------------------
|
@@ -56,24 +59,27 @@ RSpec.configure do |config|
|
|
56
59
|
# leader: coordinator.stub
|
57
60
|
# datasources: datasource1, datasource2
|
58
61
|
# tiers: _default_tier, hot
|
59
|
-
|
60
|
-
#
|
61
|
-
|
62
|
+
|
63
|
+
# Load data
|
64
|
+
responses = YAML.load(File.read('spec/data/druid_responses.yml'))
|
65
|
+
head = { 'Accept' => '*/*', 'User-Agent' => 'Ruby' }
|
66
|
+
rhead = { 'Content-Type' => 'application/json' }
|
67
|
+
base = 'http://coordinator.stub/druid/coordinator/v1/'
|
68
|
+
|
69
|
+
# Stub all queries to the API
|
70
|
+
stub_request(:get, "#{base}leader").with(headers: head)
|
71
|
+
.to_return(status: 200, body: responses['leader'], headers: {})
|
62
72
|
|
63
|
-
|
64
|
-
|
65
|
-
# .to_return(status: 200, body: '{"datasource1":100.0,"datasource2":100.0}',
|
66
|
-
# headers: { 'Content-Type' => 'application/json' })
|
73
|
+
stub_request(:get, "#{base}loadstatus").with(headers: head)
|
74
|
+
.to_return(status: 200, body: responses['loadstatus'], headers: rhead)
|
67
75
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
# headers: { 'Content-Type' => 'application/json' })
|
76
|
+
stub_request(:get, "#{base}loadstatus?simple").with(headers: head)
|
77
|
+
.to_return(status: 200, body: responses['loadstatus_simple'],
|
78
|
+
headers: rhead)
|
72
79
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
# headers: { 'Content-Type' => 'application/json' })
|
80
|
+
stub_request(:get, "#{base}loadstatus?full").with(headers: head)
|
81
|
+
.to_return(status: 200, body: responses['loadstatus_full'],
|
82
|
+
headers: rhead)
|
77
83
|
end
|
78
84
|
end
|
79
85
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: druid_config
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Angel M Miguel
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-10-
|
11
|
+
date: 2015-10-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: zk
|
@@ -152,12 +152,14 @@ files:
|
|
152
152
|
- lib/druid_config/entities/node.rb
|
153
153
|
- lib/druid_config/entities/rule.rb
|
154
154
|
- lib/druid_config/entities/segment.rb
|
155
|
+
- lib/druid_config/entities/task.rb
|
155
156
|
- lib/druid_config/entities/tier.rb
|
156
157
|
- lib/druid_config/entities/worker.rb
|
157
158
|
- lib/druid_config/util.rb
|
158
159
|
- lib/druid_config/version.rb
|
159
160
|
- lib/druid_config/zk.rb
|
160
161
|
- spec/cluster_spec.rb
|
162
|
+
- spec/data/druid_responses.yml
|
161
163
|
- spec/data_source_spec.rb
|
162
164
|
- spec/node_spec.rb
|
163
165
|
- spec/spec_helper.rb
|
@@ -187,6 +189,7 @@ specification_version: 4
|
|
187
189
|
summary: Fetch Druid info and configure your cluster
|
188
190
|
test_files:
|
189
191
|
- spec/cluster_spec.rb
|
192
|
+
- spec/data/druid_responses.yml
|
190
193
|
- spec/data_source_spec.rb
|
191
194
|
- spec/node_spec.rb
|
192
195
|
- spec/spec_helper.rb
|