druid_config 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +12 -6
- data/lib/druid_config.rb +1 -0
- data/lib/druid_config/cluster.rb +66 -30
- data/lib/druid_config/entities/data_source.rb +26 -13
- data/lib/druid_config/entities/node.rb +65 -13
- data/lib/druid_config/entities/task.rb +97 -0
- data/lib/druid_config/entities/tier.rb +18 -4
- data/lib/druid_config/entities/worker.rb +12 -4
- data/lib/druid_config/util.rb +34 -0
- data/lib/druid_config/version.rb +1 -1
- data/lib/druid_config/zk.rb +6 -0
- data/spec/cluster_spec.rb +20 -24
- data/spec/data/druid_responses.yml +4 -0
- data/spec/node_spec.rb +6 -4
- data/spec/spec_helper.rb +22 -16
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a03eb8b4f7df4223fe098516a23443beac52be00
|
4
|
+
data.tar.gz: 230450c3d105e2327c681f1bc1beb225926361be
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 151efb283126cbf585549a0e1d0a7f766f48c51e3f21e18ab373c34d59a247c64fbcc1188157ccaadb7b5940ccdcc480c24b87f33a2f0e890e019be817b9bcab
|
7
|
+
data.tar.gz: db35a7cc044f47bb4de83ea1467e35186389471485f71f48e3f4363087dc8f9326150a0df93a7844d901b804bf38348d51ab2cfb452a2aa708240cbd08a5ba76
|
data/README.md
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
[![Gem Version](https://badge.fury.io/rb/druid_config.svg)](https://badge.fury.io/rb/druid_config) [![Build Status](https://travis-ci.org/redBorder/druid_config.svg)](https://travis-ci.org/redBorder/druid_config)
|
2
|
+
|
1
3
|
# DruidConfig
|
2
4
|
|
3
5
|
DruidConfig is a gem to access the information about Druid cluster status. You can check a node capacity, number of segments, tiers... It uses [zookeeper](https://zookeeper.apache.org/) to get coordinator and overlord URIs.
|
@@ -5,9 +7,13 @@ DruidConfig is a gem to access the information about Druid cluster status. You c
|
|
5
7
|
To use in your application, add this line to your Gemfile:
|
6
8
|
|
7
9
|
```ruby
|
8
|
-
|
10
|
+
gem 'druid_config'
|
9
11
|
```
|
10
12
|
|
13
|
+
## Query Druid data
|
14
|
+
|
15
|
+
The purpose of this gem is to query Druid cluster status. If you want to query Druid data, I recommend you to use [ruby-druid gem](https://github.com/ruby-druid/ruby-druid).
|
16
|
+
|
11
17
|
# Initialization
|
12
18
|
|
13
19
|
`Cluster` is the base class to perform queries. To initialize it send the Zookeeper URI and options as arguments:
|
@@ -44,11 +50,11 @@ Call methods defined in `DruidConfig::Cluster` to access to the data. To get mor
|
|
44
50
|
|
45
51
|
Some methods return an instance of an `Entity` class. These entities provide multiple methods to access data. Defined entities are inside `druid_config/entities` folder.
|
46
52
|
|
47
|
-
* [DataSource]()
|
48
|
-
* [
|
49
|
-
* [
|
50
|
-
* [
|
51
|
-
* [Worker]()
|
53
|
+
* [DataSource](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/data_source.rb)
|
54
|
+
* [Segment](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/segment.rb)
|
55
|
+
* [Tier](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/tier.rb)
|
56
|
+
* [Node](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/node.rb)
|
57
|
+
* [Worker](https://github.com/redBorder/druid_config/blob/master/lib/druid_config/entities/worker.rb)
|
52
58
|
|
53
59
|
## Exceptions
|
54
60
|
|
data/lib/druid_config.rb
CHANGED
@@ -6,6 +6,7 @@ require 'druid_config/zk'
|
|
6
6
|
require 'druid_config/version'
|
7
7
|
require 'druid_config/util'
|
8
8
|
require 'druid_config/entities/segment'
|
9
|
+
require 'druid_config/entities/task'
|
9
10
|
require 'druid_config/entities/worker'
|
10
11
|
require 'druid_config/entities/node'
|
11
12
|
require 'druid_config/entities/tier'
|
data/lib/druid_config/cluster.rb
CHANGED
@@ -136,7 +136,7 @@ module DruidConfig
|
|
136
136
|
def datasources
|
137
137
|
datasource_status = load_status
|
138
138
|
secure_query do
|
139
|
-
self.class.get('/datasources?
|
139
|
+
self.class.get('/datasources?simple').map do |data|
|
140
140
|
DruidConfig::Entities::DataSource.new(
|
141
141
|
data,
|
142
142
|
datasource_status.select { |k, _| k == data['name'] }.values.first)
|
@@ -202,8 +202,8 @@ module DruidConfig
|
|
202
202
|
#
|
203
203
|
def servers
|
204
204
|
secure_query do
|
205
|
-
queue = load_queue('
|
206
|
-
self.class.get('/servers?
|
205
|
+
queue = load_queue('simple')
|
206
|
+
self.class.get('/servers?simple').map do |data|
|
207
207
|
DruidConfig::Entities::Node.new(
|
208
208
|
data,
|
209
209
|
queue.select { |k, _| k == data['host'] }.values.first)
|
@@ -253,22 +253,14 @@ module DruidConfig
|
|
253
253
|
# Array of Workers
|
254
254
|
#
|
255
255
|
def workers
|
256
|
-
# Stash the base_uri
|
257
|
-
stash_uri
|
258
|
-
self.class.base_uri(
|
259
|
-
"#{DruidConfig.client.overlord}"\
|
260
|
-
"druid/indexer/#{DruidConfig::Version::API_VERSION}")
|
261
256
|
workers = []
|
262
257
|
# Perform a query
|
263
|
-
|
258
|
+
query_overlord do
|
264
259
|
secure_query do
|
265
260
|
workers = self.class.get('/workers').map do |worker|
|
266
261
|
DruidConfig::Entities::Worker.new(worker)
|
267
262
|
end
|
268
263
|
end
|
269
|
-
ensure
|
270
|
-
# Recover it
|
271
|
-
pop_uri
|
272
264
|
end
|
273
265
|
# Return
|
274
266
|
workers
|
@@ -281,6 +273,68 @@ module DruidConfig
|
|
281
273
|
@physical_workers ||= workers.map(&:host).uniq
|
282
274
|
end
|
283
275
|
|
276
|
+
# Tasks
|
277
|
+
# -----------------
|
278
|
+
|
279
|
+
#
|
280
|
+
# Return tasks based on the status
|
281
|
+
#
|
282
|
+
# == Returns:
|
283
|
+
# Array of Tasks
|
284
|
+
#
|
285
|
+
%w(running pending waiting).each do |status|
|
286
|
+
define_method("#{status}_tasks") do
|
287
|
+
tasks = []
|
288
|
+
query_overlord do
|
289
|
+
tasks = self.class.get("/#{status}Tasks").map do |task|
|
290
|
+
DruidConfig::Entities::Task.new(
|
291
|
+
task['id'],
|
292
|
+
DruidConfig::Entities::Task::STATUS[status.to_sym],
|
293
|
+
created_time: task['createdTime'],
|
294
|
+
query_insertion_time: task['queueInsertionTime'])
|
295
|
+
end
|
296
|
+
end
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
300
|
+
#
|
301
|
+
# Find a task
|
302
|
+
#
|
303
|
+
def task(id)
|
304
|
+
DruidConfig::Entities::Task.find(id)
|
305
|
+
end
|
306
|
+
|
307
|
+
#
|
308
|
+
# Return complete tasks
|
309
|
+
#
|
310
|
+
# == Returns:
|
311
|
+
# Array of Tasks
|
312
|
+
#
|
313
|
+
def complete_tasks
|
314
|
+
tasks = []
|
315
|
+
query_overlord do
|
316
|
+
tasks = self.class.get('/completeTasks').map do |task|
|
317
|
+
DruidConfig::Entities::Task.new(
|
318
|
+
task['id'],
|
319
|
+
task['statusCode'])
|
320
|
+
end
|
321
|
+
end
|
322
|
+
end
|
323
|
+
|
324
|
+
#
|
325
|
+
# Return failed completed tasks
|
326
|
+
#
|
327
|
+
def failed_tasks
|
328
|
+
complete_tasks.select(&:failed?)
|
329
|
+
end
|
330
|
+
|
331
|
+
#
|
332
|
+
# Return success completed tasks
|
333
|
+
#
|
334
|
+
def success_tasks
|
335
|
+
complete_tasks.select(&:success?)
|
336
|
+
end
|
337
|
+
|
284
338
|
# Services
|
285
339
|
# -----------------
|
286
340
|
|
@@ -302,23 +356,5 @@ module DruidConfig
|
|
302
356
|
# Return nodes
|
303
357
|
@services = services
|
304
358
|
end
|
305
|
-
|
306
|
-
private
|
307
|
-
|
308
|
-
#
|
309
|
-
# Stash current base_uri
|
310
|
-
#
|
311
|
-
def stash_uri
|
312
|
-
@uri_stack ||= []
|
313
|
-
@uri_stack.push self.class.base_uri
|
314
|
-
end
|
315
|
-
|
316
|
-
#
|
317
|
-
# Pop next base_uri
|
318
|
-
#
|
319
|
-
def pop_uri
|
320
|
-
return if @uri_stack.nil? || @uri_stack.empty?
|
321
|
-
self.class.base_uri(@uri_stack.pop)
|
322
|
-
end
|
323
359
|
end
|
324
360
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
module DruidConfig
|
2
2
|
#
|
3
|
-
# Module of
|
3
|
+
# Module of Entities
|
4
4
|
#
|
5
5
|
module Entities
|
6
6
|
#
|
@@ -8,6 +8,7 @@ module DruidConfig
|
|
8
8
|
class DataSource
|
9
9
|
# HTTParty Rocks!
|
10
10
|
include HTTParty
|
11
|
+
include DruidConfig::Util
|
11
12
|
|
12
13
|
attr_reader :name, :properties, :load_status
|
13
14
|
|
@@ -31,28 +32,36 @@ module DruidConfig
|
|
31
32
|
# http://druid.io/docs/0.8.1/design/coordinator.html
|
32
33
|
#
|
33
34
|
|
34
|
-
def info
|
35
|
-
|
35
|
+
def info
|
36
|
+
secure_query do
|
37
|
+
@info ||= self.class.get("/datasources/#{@name}")
|
38
|
+
end
|
36
39
|
end
|
37
40
|
|
38
41
|
# Intervals
|
39
42
|
# -----------------
|
40
43
|
def intervals(params = '')
|
41
|
-
|
44
|
+
secure_query do
|
45
|
+
self.class.get("/datasources/#{@name}/intervals?#{params}")
|
46
|
+
end
|
42
47
|
end
|
43
48
|
|
44
49
|
def interval(interval, params = '')
|
45
|
-
|
46
|
-
|
50
|
+
secure_query do
|
51
|
+
self.class.get("/datasources/#{@name}/intervals/#{interval}"\
|
52
|
+
"?#{params}")
|
53
|
+
end
|
47
54
|
end
|
48
55
|
|
49
56
|
# Segments and Tiers
|
50
57
|
# -----------------
|
51
58
|
def segments
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
59
|
+
secure_query do
|
60
|
+
@segments ||=
|
61
|
+
self.class.get("/datasources/#{@name}/segments?full").map do |s|
|
62
|
+
DruidConfig::Entities::Segment.new(s)
|
63
|
+
end
|
64
|
+
end
|
56
65
|
end
|
57
66
|
|
58
67
|
def segment(segment)
|
@@ -66,12 +75,16 @@ module DruidConfig
|
|
66
75
|
# Rules
|
67
76
|
# -----------------
|
68
77
|
def rules(params = '')
|
69
|
-
|
78
|
+
secure_query do
|
79
|
+
self.class.get("/rules/#{@name}?#{params}")
|
80
|
+
end
|
70
81
|
end
|
71
82
|
|
72
83
|
def history_rules(interval)
|
73
|
-
|
74
|
-
|
84
|
+
secure_query do
|
85
|
+
self.class.get("/rules/#{@name}/history"\
|
86
|
+
"?interval=#{interval}")
|
87
|
+
end
|
75
88
|
end
|
76
89
|
end
|
77
90
|
end
|
@@ -6,10 +6,11 @@ module DruidConfig
|
|
6
6
|
class Node
|
7
7
|
# HTTParty Rocks!
|
8
8
|
include HTTParty
|
9
|
+
include DruidConfig::Util
|
9
10
|
|
10
11
|
# Readers
|
11
12
|
attr_reader :host, :port, :max_size, :type, :tier, :priority, :size,
|
12
|
-
:
|
13
|
+
:segments_to_load_count, :segments_to_drop_count,
|
13
14
|
:segments_to_load_size, :segments_to_drop_size
|
14
15
|
|
15
16
|
#
|
@@ -28,21 +29,20 @@ module DruidConfig
|
|
28
29
|
@tier = metadata['tier']
|
29
30
|
@priority = metadata['priority']
|
30
31
|
@size = metadata['currSize']
|
31
|
-
|
32
|
-
|
33
|
-
|
32
|
+
# Set end point for HTTParty
|
33
|
+
self.class.base_uri(
|
34
|
+
"#{DruidConfig.client.coordinator}"\
|
35
|
+
"druid/coordinator/#{DruidConfig::Version::API_VERSION}")
|
36
|
+
|
37
|
+
# Load more data from queue
|
34
38
|
if queue.nil?
|
35
|
-
@
|
39
|
+
@segments_to_load_count, @segments_to_drop_count = 0, 0
|
36
40
|
@segments_to_load_size, @segments_to_drop_size = 0, 0
|
37
41
|
else
|
38
|
-
@
|
39
|
-
|
40
|
-
|
41
|
-
@
|
42
|
-
DruidConfig::Entities::Segment.new(segment)
|
43
|
-
end
|
44
|
-
@segments_to_load_size = @segments_to_load.map(&:size).reduce(:+)
|
45
|
-
@segments_to_drop_size = @segments_to_drop.map(&:size).reduce(:+)
|
42
|
+
@segments_to_load_count = queue['segmentsToLoad']
|
43
|
+
@segments_to_drop_count = queue['segmentsToDrop']
|
44
|
+
@segments_to_load_size = queue['segmentsToLoadSize']
|
45
|
+
@segments_to_drop_size = queue['segmentsToLoadSize']
|
46
46
|
end
|
47
47
|
end
|
48
48
|
|
@@ -63,12 +63,64 @@ module DruidConfig
|
|
63
63
|
max_size - size
|
64
64
|
end
|
65
65
|
|
66
|
+
#
|
67
|
+
# Return all segments of this node
|
68
|
+
#
|
69
|
+
def segments_count
|
70
|
+
@segments_count ||=
|
71
|
+
self.class.get("/servers/#{uri}/segments").size
|
72
|
+
end
|
73
|
+
|
74
|
+
#
|
75
|
+
# Return all segments of this node
|
76
|
+
#
|
77
|
+
def segments
|
78
|
+
@segments ||=
|
79
|
+
self.class.get("/servers/#{uri}/segments?full").map do |s|
|
80
|
+
DruidConfig::Entities::Segment.new(s)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
#
|
85
|
+
# Get segments to load
|
86
|
+
#
|
87
|
+
def segments_to_load
|
88
|
+
current_queue = queue
|
89
|
+
return [] unless current_queue
|
90
|
+
current_queue['segmentsToLoad'].map do |segment|
|
91
|
+
DruidConfig::Entities::Segment.new(segment)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
#
|
96
|
+
# Get segments to drop
|
97
|
+
#
|
98
|
+
def segments_to_drop
|
99
|
+
current_queue = queue
|
100
|
+
return [] unless current_queue
|
101
|
+
current_queue['segmentsToDrop'].map do |segment|
|
102
|
+
DruidConfig::Entities::Segment.new(segment)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
66
106
|
#
|
67
107
|
# Return the URI of this node
|
68
108
|
#
|
69
109
|
def uri
|
70
110
|
"#{@host}:#{@port}"
|
71
111
|
end
|
112
|
+
|
113
|
+
private
|
114
|
+
|
115
|
+
#
|
116
|
+
# Get load queue for this node
|
117
|
+
#
|
118
|
+
def queue
|
119
|
+
secure_query do
|
120
|
+
self.class.get('/loadqueue?full')
|
121
|
+
.select { |k, _| k == uri }.values.first
|
122
|
+
end
|
123
|
+
end
|
72
124
|
end
|
73
125
|
end
|
74
126
|
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
module DruidConfig
|
2
|
+
#
|
3
|
+
# Module of Entities
|
4
|
+
#
|
5
|
+
module Entities
|
6
|
+
#
|
7
|
+
# Init a DataSource
|
8
|
+
class Task
|
9
|
+
# HTTParty Rocks!
|
10
|
+
include HTTParty
|
11
|
+
include DruidConfig::Util
|
12
|
+
|
13
|
+
# Statuses constants
|
14
|
+
STATUS = {
|
15
|
+
running: 'RUNNING',
|
16
|
+
pending: 'PENDING',
|
17
|
+
success: 'SUCCESS',
|
18
|
+
waiting: 'WAITING',
|
19
|
+
failed: 'FAILED'
|
20
|
+
}
|
21
|
+
|
22
|
+
attr_reader :id, :status, :created_time, :query_insertion_time
|
23
|
+
|
24
|
+
#
|
25
|
+
# Find a task based on id
|
26
|
+
#
|
27
|
+
def self.find(id)
|
28
|
+
# Set end point for HTTParty
|
29
|
+
base_uri(
|
30
|
+
"#{DruidConfig.client.overlord}"\
|
31
|
+
"druid/indexer/#{DruidConfig::Version::API_VERSION}/task")
|
32
|
+
# Get data
|
33
|
+
status = get("/#{id}/status")
|
34
|
+
new(id, status['status']['status'])
|
35
|
+
end
|
36
|
+
|
37
|
+
#
|
38
|
+
# Initialize a task
|
39
|
+
#
|
40
|
+
# == Parameters:
|
41
|
+
# id::
|
42
|
+
# String with identifier
|
43
|
+
# status::
|
44
|
+
# Current status of the task
|
45
|
+
#
|
46
|
+
def initialize(id, status, extended_info = {})
|
47
|
+
@id = id
|
48
|
+
@status = status.downcase.to_sym
|
49
|
+
@created_time = extended_info[:created_time]
|
50
|
+
@query_insertion_time = extended_info[:query_insertion_time]
|
51
|
+
# Set end point for HTTParty
|
52
|
+
self.class.base_uri(
|
53
|
+
"#{DruidConfig.client.overlord}"\
|
54
|
+
"druid/indexer/#{DruidConfig::Version::API_VERSION}/task")
|
55
|
+
end
|
56
|
+
|
57
|
+
#
|
58
|
+
# Multiple methods to check status of the tasks
|
59
|
+
#
|
60
|
+
STATUS.keys.each do |s|
|
61
|
+
define_method("#{s}?") do
|
62
|
+
s.to_sym == @status
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
alias_method :completed?, :success?
|
67
|
+
|
68
|
+
#
|
69
|
+
# Get payload of the task
|
70
|
+
#
|
71
|
+
def payload
|
72
|
+
@payload ||= self.class.get("/#{@id}")['payload']
|
73
|
+
end
|
74
|
+
|
75
|
+
#
|
76
|
+
# Current log of a task
|
77
|
+
#
|
78
|
+
def log
|
79
|
+
self.class.get("/#{@id}/log")
|
80
|
+
end
|
81
|
+
|
82
|
+
#
|
83
|
+
# Get the dataSource of this task
|
84
|
+
#
|
85
|
+
def datasource
|
86
|
+
payload['dataSource']
|
87
|
+
end
|
88
|
+
|
89
|
+
#
|
90
|
+
# Group of the task
|
91
|
+
#
|
92
|
+
def group_id
|
93
|
+
payload['groupId']
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -42,14 +42,28 @@ module DruidConfig
|
|
42
42
|
.flatten.sort_by { |seg| seg.interval.first }
|
43
43
|
end
|
44
44
|
|
45
|
+
def segments_count
|
46
|
+
@segments_count ||= nodes.map(&:segments_count).inject(:+)
|
47
|
+
end
|
48
|
+
|
45
49
|
def segments_to_load
|
46
|
-
@segments_to_load ||=
|
47
|
-
|
50
|
+
@segments_to_load ||= nodes.map(&:segments_to_load)
|
51
|
+
.flatten.sort_by { |seg| seg.interval.first }
|
48
52
|
end
|
49
53
|
|
50
54
|
def segments_to_drop
|
51
|
-
@segments_to_drop ||=
|
52
|
-
|
55
|
+
@segments_to_drop ||= nodes.map(&:segments_to_drop)
|
56
|
+
.flatten.sort_by { |seg| seg.interval.first }
|
57
|
+
end
|
58
|
+
|
59
|
+
def segments_to_load_count
|
60
|
+
@segments_to_load_count ||=
|
61
|
+
nodes.map(&:segments_to_load_count).inject(:+)
|
62
|
+
end
|
63
|
+
|
64
|
+
def segments_to_drop_count
|
65
|
+
@segments_to_drop_count ||=
|
66
|
+
nodes.map(&:segments_to_drop_count).inject(:+)
|
53
67
|
end
|
54
68
|
|
55
69
|
def segments_to_load_size
|
@@ -6,7 +6,7 @@ module DruidConfig
|
|
6
6
|
class Worker
|
7
7
|
# Readers
|
8
8
|
attr_reader :last_completed_task_time, :host, :port, :ip, :capacity,
|
9
|
-
|
9
|
+
:version, :running_tasks, :capacity_used
|
10
10
|
|
11
11
|
#
|
12
12
|
# Initialize it with received info
|
@@ -21,21 +21,29 @@ module DruidConfig
|
|
21
21
|
@capacity = metadata['worker']['capacity']
|
22
22
|
@version = metadata['worker']['version']
|
23
23
|
@last_completed_task_time = metadata['lastCompletedTaskTime']
|
24
|
-
@running_tasks = metadata['runningTasks']
|
24
|
+
@running_tasks = metadata['runningTasks'].map do |task|
|
25
|
+
DruidConfig::Entities::Task.new(
|
26
|
+
task,
|
27
|
+
DruidConfig::Entities::Task::STATUS[:running],
|
28
|
+
created_time: task['createdTime'],
|
29
|
+
query_insertion_time: task['queueInsertionTime'])
|
30
|
+
end
|
25
31
|
@capacity_used = metadata['currCapacityUsed']
|
26
32
|
end
|
27
33
|
|
34
|
+
alias_method :used, :capacity_used
|
35
|
+
|
28
36
|
#
|
29
37
|
# Return free capacity
|
30
38
|
#
|
31
39
|
def free
|
32
|
-
@free ||= (capacity -
|
40
|
+
@free ||= (capacity - capacity_used)
|
33
41
|
end
|
34
42
|
|
35
43
|
#
|
36
44
|
# Return capacity used
|
37
45
|
#
|
38
|
-
def
|
46
|
+
def used_percent
|
39
47
|
return 0 unless @capacity && @capacity != 0
|
40
48
|
((@capacity_used.to_f / @capacity) * 100).round(2)
|
41
49
|
end
|
data/lib/druid_config/util.rb
CHANGED
@@ -31,5 +31,39 @@ module DruidConfig
|
|
31
31
|
retry
|
32
32
|
end
|
33
33
|
end
|
34
|
+
|
35
|
+
#
|
36
|
+
# Update the URI of HTTParty to perform queries to Overlord.
|
37
|
+
# After perform the query, the URI is reverted to coordinator.
|
38
|
+
#
|
39
|
+
def query_overlord
|
40
|
+
return unless block_given?
|
41
|
+
stash_uri
|
42
|
+
self.class.base_uri(
|
43
|
+
"#{DruidConfig.client.overlord}"\
|
44
|
+
"druid/indexer/#{DruidConfig::Version::API_VERSION}")
|
45
|
+
begin
|
46
|
+
yield
|
47
|
+
ensure
|
48
|
+
# Ensure we revert the URI
|
49
|
+
pop_uri
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
#
|
54
|
+
# Stash current base_uri
|
55
|
+
#
|
56
|
+
def stash_uri
|
57
|
+
@uri_stack ||= []
|
58
|
+
@uri_stack.push self.class.base_uri
|
59
|
+
end
|
60
|
+
|
61
|
+
#
|
62
|
+
# Pop next base_uri
|
63
|
+
#
|
64
|
+
def pop_uri
|
65
|
+
return if @uri_stack.nil? || @uri_stack.empty?
|
66
|
+
self.class.base_uri(@uri_stack.pop)
|
67
|
+
end
|
34
68
|
end
|
35
69
|
end
|
data/lib/druid_config/version.rb
CHANGED
data/lib/druid_config/zk.rb
CHANGED
data/spec/cluster_spec.rb
CHANGED
@@ -2,31 +2,27 @@ require 'spec_helper'
|
|
2
2
|
require 'pry'
|
3
3
|
require 'pry-nav'
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
5
|
+
describe DruidConfig::Cluster do
|
6
|
+
before(:each) do
|
7
|
+
@cluster = DruidConfig::Cluster.new('localhost', zk_keepalive: true)
|
8
|
+
end
|
9
9
|
|
10
|
-
|
11
|
-
|
12
|
-
|
10
|
+
it 'must get the leader' do
|
11
|
+
expect(@cluster.leader).to eq 'coordinator.stub'
|
12
|
+
end
|
13
13
|
|
14
|
-
|
15
|
-
|
14
|
+
it 'must get load status' do
|
15
|
+
basic = @cluster.load_status
|
16
|
+
expect(basic.keys).to eq %w(datasource1 datasource2)
|
17
|
+
expect(basic[basic.keys.first]).to eq 100
|
16
18
|
|
19
|
+
simple = @cluster.load_status('simple')
|
20
|
+
expect(simple.keys).to eq %w(datasource1 datasource2)
|
21
|
+
expect(simple[simple.keys.first]).to eq 0
|
17
22
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
# # expect(simple[simple.keys.first]).to eq 0
|
25
|
-
|
26
|
-
# # # Use tiers
|
27
|
-
# # simple = @cluster.load_status('full')
|
28
|
-
# # expect(simple.keys).to eq %w(_default_tier hot)
|
29
|
-
# # expect(simple['_default_tier']['datasource1']).to eq 0
|
30
|
-
# # expect(simple['hot']['datasource2']).to eq 0
|
31
|
-
# end
|
32
|
-
# end
|
23
|
+
full = @cluster.load_status('full')
|
24
|
+
expect(full.keys).to eq %w(_default_tier hot)
|
25
|
+
expect(full['_default_tier']['datasource1']).to eq 0
|
26
|
+
expect(full['hot']['datasource2']).to eq 0
|
27
|
+
end
|
28
|
+
end
|
data/spec/node_spec.rb
CHANGED
@@ -28,8 +28,10 @@ describe DruidConfig::Entities::Node do
|
|
28
28
|
'priority' => 0, 'segments' => @segments, 'currSize' => @size }
|
29
29
|
|
30
30
|
@queue = {
|
31
|
-
'segmentsToLoad' =>
|
32
|
-
'segmentsToDrop' =>
|
31
|
+
'segmentsToLoad' => 0,
|
32
|
+
'segmentsToDrop' => 0,
|
33
|
+
'segmentsToLoadSize' => 0,
|
34
|
+
'segmentsToDropSize' => 0
|
33
35
|
}
|
34
36
|
end
|
35
37
|
|
@@ -41,8 +43,8 @@ describe DruidConfig::Entities::Node do
|
|
41
43
|
expect(datasource.type).to eq @type.to_sym
|
42
44
|
expect(datasource.priority).to eq @priority
|
43
45
|
expect(datasource.size).to eq @size
|
44
|
-
expect(datasource.
|
45
|
-
expect(datasource.
|
46
|
+
expect(datasource.segments_to_load_count).to eq 0
|
47
|
+
expect(datasource.segments_to_drop_count).to eq 0
|
46
48
|
end
|
47
49
|
|
48
50
|
it 'calculate free space' do
|
data/spec/spec_helper.rb
CHANGED
@@ -46,8 +46,11 @@ RSpec.configure do |config|
|
|
46
46
|
def coordinator
|
47
47
|
'coordinator.stub/'
|
48
48
|
end
|
49
|
+
def overlord
|
50
|
+
'overlord.stub/'
|
51
|
+
end
|
49
52
|
end
|
50
|
-
allow(DruidConfig).to receive(:
|
53
|
+
allow(DruidConfig::Client).to receive(:new) { ClientStub.new }
|
51
54
|
|
52
55
|
# Stub queries
|
53
56
|
# ----------------------------------
|
@@ -56,24 +59,27 @@ RSpec.configure do |config|
|
|
56
59
|
# leader: coordinator.stub
|
57
60
|
# datasources: datasource1, datasource2
|
58
61
|
# tiers: _default_tier, hot
|
59
|
-
|
60
|
-
#
|
61
|
-
|
62
|
+
|
63
|
+
# Load data
|
64
|
+
responses = YAML.load(File.read('spec/data/druid_responses.yml'))
|
65
|
+
head = { 'Accept' => '*/*', 'User-Agent' => 'Ruby' }
|
66
|
+
rhead = { 'Content-Type' => 'application/json' }
|
67
|
+
base = 'http://coordinator.stub/druid/coordinator/v1/'
|
68
|
+
|
69
|
+
# Stub all queries to the API
|
70
|
+
stub_request(:get, "#{base}leader").with(headers: head)
|
71
|
+
.to_return(status: 200, body: responses['leader'], headers: {})
|
62
72
|
|
63
|
-
|
64
|
-
|
65
|
-
# .to_return(status: 200, body: '{"datasource1":100.0,"datasource2":100.0}',
|
66
|
-
# headers: { 'Content-Type' => 'application/json' })
|
73
|
+
stub_request(:get, "#{base}loadstatus").with(headers: head)
|
74
|
+
.to_return(status: 200, body: responses['loadstatus'], headers: rhead)
|
67
75
|
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
# headers: { 'Content-Type' => 'application/json' })
|
76
|
+
stub_request(:get, "#{base}loadstatus?simple").with(headers: head)
|
77
|
+
.to_return(status: 200, body: responses['loadstatus_simple'],
|
78
|
+
headers: rhead)
|
72
79
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
# headers: { 'Content-Type' => 'application/json' })
|
80
|
+
stub_request(:get, "#{base}loadstatus?full").with(headers: head)
|
81
|
+
.to_return(status: 200, body: responses['loadstatus_full'],
|
82
|
+
headers: rhead)
|
77
83
|
end
|
78
84
|
end
|
79
85
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: druid_config
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Angel M Miguel
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-10-
|
11
|
+
date: 2015-10-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: zk
|
@@ -152,12 +152,14 @@ files:
|
|
152
152
|
- lib/druid_config/entities/node.rb
|
153
153
|
- lib/druid_config/entities/rule.rb
|
154
154
|
- lib/druid_config/entities/segment.rb
|
155
|
+
- lib/druid_config/entities/task.rb
|
155
156
|
- lib/druid_config/entities/tier.rb
|
156
157
|
- lib/druid_config/entities/worker.rb
|
157
158
|
- lib/druid_config/util.rb
|
158
159
|
- lib/druid_config/version.rb
|
159
160
|
- lib/druid_config/zk.rb
|
160
161
|
- spec/cluster_spec.rb
|
162
|
+
- spec/data/druid_responses.yml
|
161
163
|
- spec/data_source_spec.rb
|
162
164
|
- spec/node_spec.rb
|
163
165
|
- spec/spec_helper.rb
|
@@ -187,6 +189,7 @@ specification_version: 4
|
|
187
189
|
summary: Fetch Druid info and configure your cluster
|
188
190
|
test_files:
|
189
191
|
- spec/cluster_spec.rb
|
192
|
+
- spec/data/druid_responses.yml
|
190
193
|
- spec/data_source_spec.rb
|
191
194
|
- spec/node_spec.rb
|
192
195
|
- spec/spec_helper.rb
|