hotdog 0.26.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/hotdog/application.rb +61 -7
- data/lib/hotdog/commands/down.rb +3 -3
- data/lib/hotdog/commands/search.rb +9 -1
- data/lib/hotdog/commands/ssh.rb +14 -12
- data/lib/hotdog/commands.rb +54 -36
- data/lib/hotdog/expression/semantics.rb +162 -163
- data/lib/hotdog/version.rb +1 -1
- data/spec/core/commands_spec.rb +21 -0
- data/spec/optimizer/binary_expression_spec.rb +36 -34
- data/spec/optimizer/glob_expression_spec.rb +93 -91
- data/spec/optimizer/regexp_expression_spec.rb +43 -41
- data/spec/optimizer/string_expression_spec.rb +93 -91
- data/spec/optimizer/unary_expression_spec.rb +258 -257
- data/spec/spec_helper.rb +6 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ed4a83ac54e12aecba688f7f0bb7b563f9cee80c
|
4
|
+
data.tar.gz: '038bbcb0dbc3e5e541c9a433ec64946beec79b62'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 35816f0033e867223cd699e3740566339896d6bf53e08f5e140c245e6e4b23abb7eebbaf996ff69c096e096ff1f9b2acab24b72700198c4063cbd911adffbbc9
|
7
|
+
data.tar.gz: d4158608de22ab5bee02e542bedf87f5e1fe2b2dac6b5bb82e02529930763830c0c371789ad1cd3fdec2adb634c0ad1c7913d427c57287d012ad0d07b2580421
|
data/lib/hotdog/application.rb
CHANGED
@@ -11,11 +11,32 @@ require "hotdog/version"
|
|
11
11
|
module Hotdog
|
12
12
|
SQLITE_LIMIT_COMPOUND_SELECT = 500 # TODO: get actual value from `sqlite3_limit()`?
|
13
13
|
|
14
|
+
# only datadog is supported as of Sep 5, 2017
|
15
|
+
SOURCE_DATADOG = 0x01
|
16
|
+
|
17
|
+
# | status | description |
|
18
|
+
# | -------- | ------------- |
|
19
|
+
# | 00000000 | pending |
|
20
|
+
# | 00010000 | running |
|
21
|
+
# | 00100000 | shutting-down |
|
22
|
+
# | 00110000 | terminated |
|
23
|
+
# | 01000000 | stopping |
|
24
|
+
# | 01010000 | stopped |
|
25
|
+
STATUS_PENDING = 0b00000000
|
26
|
+
STATUS_RUNNING = 0b00010000
|
27
|
+
STATUS_SHUTTING_DOWN = 0b00100000
|
28
|
+
STATUS_TERMINATED = 0b00110000
|
29
|
+
STATUS_STOPPING = 0b01000000
|
30
|
+
STATUS_STOPPED = 0b01010000
|
31
|
+
|
32
|
+
VERBOSITY_NULL = 0
|
33
|
+
VERBOSITY_INFO = 1
|
34
|
+
VERBOSITY_DEBUG = 2
|
35
|
+
VERBOSITY_TRACE = 4
|
36
|
+
|
14
37
|
class Application
|
15
38
|
def initialize()
|
16
|
-
@logger = Logger.new(STDERR)
|
17
|
-
logger.level = Logger::INFO
|
18
|
-
}
|
39
|
+
@logger = Logger.new(STDERR)
|
19
40
|
@optparse = OptionParser.new
|
20
41
|
@optparse.version = Hotdog::VERSION
|
21
42
|
@options = {
|
@@ -30,6 +51,7 @@ module Hotdog
|
|
30
51
|
force: false,
|
31
52
|
format: "text",
|
32
53
|
headers: false,
|
54
|
+
status: STATUS_RUNNING,
|
33
55
|
listing: false,
|
34
56
|
logger: @logger,
|
35
57
|
max_time: 5,
|
@@ -41,7 +63,9 @@ module Hotdog
|
|
41
63
|
tags: [],
|
42
64
|
display_search_tags: false,
|
43
65
|
verbose: false,
|
66
|
+
verbosity: VERBOSITY_NULL,
|
44
67
|
}
|
68
|
+
|
45
69
|
define_options
|
46
70
|
end
|
47
71
|
attr_reader :logger
|
@@ -85,10 +109,18 @@ module Hotdog
|
|
85
109
|
|
86
110
|
options[:formatter] = get_formatter(options[:format])
|
87
111
|
|
88
|
-
if options[:debug] or options[:verbose]
|
112
|
+
if ( options[:debug] or options[:verbose] ) and ( options[:verbosity] < VERBOSITY_DEBUG )
|
113
|
+
options[:verbosity] = VERBOSITY_DEBUG
|
114
|
+
end
|
115
|
+
|
116
|
+
if VERBOSITY_DEBUG <= options[:verbosity]
|
89
117
|
options[:logger].level = Logger::DEBUG
|
90
118
|
else
|
91
|
-
options[:
|
119
|
+
if VERBOSITY_INFO <= options[:verbosity]
|
120
|
+
options[:logger].level = Logger::INFO
|
121
|
+
else
|
122
|
+
options[:logger].level = Logger::WARN
|
123
|
+
end
|
92
124
|
end
|
93
125
|
|
94
126
|
command.run(args, @options)
|
@@ -127,6 +159,10 @@ module Hotdog
|
|
127
159
|
end
|
128
160
|
end
|
129
161
|
|
162
|
+
def status()
|
163
|
+
options.fetch(:status, STATUS_RUNNING)
|
164
|
+
end
|
165
|
+
|
130
166
|
private
|
131
167
|
def define_options
|
132
168
|
@optparse.on("--endpoint ENDPOINT", "Datadog API endpoint") do |endpoint|
|
@@ -168,6 +204,24 @@ module Hotdog
|
|
168
204
|
@optparse.on("-h", "--[no-]headers", "Display headeres for each columns") do |v|
|
169
205
|
options[:headers] = v
|
170
206
|
end
|
207
|
+
@optparse.on("--status=STATUS", "Specify custom host status") do |v|
|
208
|
+
case v
|
209
|
+
when /\Apending\z/i
|
210
|
+
options[:status] = STATUS_PENDING
|
211
|
+
when /\Arunning\z/i
|
212
|
+
options[:status] = STATUS_RUNNING
|
213
|
+
when /\Ashutting-down\z/i
|
214
|
+
options[:status] = STATUS_SHUTTING_DOWN
|
215
|
+
when /\Aterminated\z/i
|
216
|
+
options[:status] = STATUS_TERMINATED
|
217
|
+
when /\Astopping\z/i
|
218
|
+
options[:status] = STATUS_STOPPING
|
219
|
+
when /\Astopped\z/i
|
220
|
+
options[:status] = STATUS_STOPPED
|
221
|
+
else
|
222
|
+
options[:status] = v.to_i
|
223
|
+
end
|
224
|
+
end
|
171
225
|
@optparse.on("-l", "--[no-]listing", "Use listing format") do |v|
|
172
226
|
options[:listing] = v
|
173
227
|
end
|
@@ -180,8 +234,8 @@ module Hotdog
|
|
180
234
|
@optparse.on("-x", "--display-search-tags", "Show tags used in search expression") do |v|
|
181
235
|
options[:display_search_tags] = v
|
182
236
|
end
|
183
|
-
@optparse.on("-V", "--[no-]verbose", "Enable verbose mode") do |v|
|
184
|
-
options[:
|
237
|
+
@optparse.on("-V", "-v", "--[no-]verbose", "Enable verbose mode") do |v|
|
238
|
+
options[:verbosity] += 1
|
185
239
|
end
|
186
240
|
@optparse.on("--[no-]offline", "Enable offline mode") do |v|
|
187
241
|
options[:offline] = v
|
data/lib/hotdog/commands/down.rb
CHANGED
@@ -40,9 +40,9 @@ module Hotdog
|
|
40
40
|
if open_db
|
41
41
|
@db.transaction do
|
42
42
|
sqlite_limit_compound_select = options[:sqlite_limit_compound_select] || SQLITE_LIMIT_COMPOUND_SELECT
|
43
|
-
hosts.each_slice(sqlite_limit_compound_select) do |hosts|
|
44
|
-
|
45
|
-
execute_db(@db,
|
43
|
+
hosts.each_slice(sqlite_limit_compound_select - 1) do |hosts|
|
44
|
+
q = "UPDATE hosts SET status = ? WHERE name IN (%s);" % hosts.map { "?" }.join(", ")
|
45
|
+
execute_db(@db, q, [STATUS_STOPPING] + hosts)
|
46
46
|
end
|
47
47
|
end
|
48
48
|
end
|
@@ -105,7 +105,15 @@ module Hotdog
|
|
105
105
|
JSON.pretty_generate(optimized.dump)
|
106
106
|
}
|
107
107
|
end
|
108
|
-
optimized.evaluate(environment)
|
108
|
+
result = optimized.evaluate(environment)
|
109
|
+
if result.empty? and !$did_reload
|
110
|
+
$did_reload = true
|
111
|
+
environment.logger.info("reloading all hosts and tags.")
|
112
|
+
environment.reload(force: true)
|
113
|
+
optimized.evaluate(environment)
|
114
|
+
else
|
115
|
+
result
|
116
|
+
end
|
109
117
|
else
|
110
118
|
raise("parser error: unknown expression: #{node.inspect}")
|
111
119
|
end
|
data/lib/hotdog/commands/ssh.rb
CHANGED
@@ -40,9 +40,6 @@ module Hotdog
|
|
40
40
|
optparse.on("-u SSH_USER", "SSH login user name") do |user|
|
41
41
|
options[:ssh_options]["User"] = user
|
42
42
|
end
|
43
|
-
optparse.on("-v", "--verbose", "Enable verbose ode") do |v|
|
44
|
-
options[:verbose] = v
|
45
|
-
end
|
46
43
|
optparse.on("--filter=COMMAND", "Command to filter search result.") do |command|
|
47
44
|
options[:filter_command] = command
|
48
45
|
end
|
@@ -71,6 +68,7 @@ module Hotdog
|
|
71
68
|
tuples, fields = get_hosts_with_search_tags(result0, node)
|
72
69
|
tuples = filter_hosts(tuples)
|
73
70
|
validate_hosts!(tuples, fields)
|
71
|
+
logger.info("target host(s): #{tuples.map {|tuple| tuple.first }.inspect}")
|
74
72
|
run_main(tuples.map {|tuple| tuple.first }, options)
|
75
73
|
end
|
76
74
|
|
@@ -125,7 +123,7 @@ module Hotdog
|
|
125
123
|
cmdline << "-F" << File.expand_path(options[:ssh_config])
|
126
124
|
end
|
127
125
|
cmdline += options[:ssh_options].flat_map { |k, v| ["-o", "#{k}=#{v}"] }
|
128
|
-
if options[:
|
126
|
+
if VERBOSITY_TRACE <= options[:verbosity]
|
129
127
|
cmdline << "-v"
|
130
128
|
end
|
131
129
|
cmdline
|
@@ -168,12 +166,14 @@ module Hotdog
|
|
168
166
|
i = 0
|
169
167
|
each_readable([cmderr, cmdout]) do |readable|
|
170
168
|
raw = readable.readline
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
169
|
+
if output
|
170
|
+
output_lock.synchronize do
|
171
|
+
if readable == cmdout
|
172
|
+
STDOUT.puts(prettify_output(raw, i, color, identifier))
|
173
|
+
i += 1
|
174
|
+
else
|
175
|
+
STDERR.puts(prettify_output(raw, nil, nil, identifier))
|
176
|
+
end
|
177
177
|
end
|
178
178
|
end
|
179
179
|
end
|
@@ -213,8 +213,10 @@ module Hotdog
|
|
213
213
|
end
|
214
214
|
buf << identifier
|
215
215
|
buf << ":"
|
216
|
-
|
217
|
-
|
216
|
+
if i
|
217
|
+
buf << i.to_s
|
218
|
+
buf << ":"
|
219
|
+
end
|
218
220
|
if color
|
219
221
|
buf << "\e[0m"
|
220
222
|
end
|
data/lib/hotdog/commands.rb
CHANGED
@@ -81,7 +81,10 @@ module Hotdog
|
|
81
81
|
end
|
82
82
|
|
83
83
|
def get_hosts(host_ids, tags=nil)
|
84
|
-
|
84
|
+
status = application.status || STATUS_RUNNING
|
85
|
+
host_ids = Array(host_ids).each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
|
86
|
+
execute("SELECT id FROM hosts WHERE status = ? AND id IN (%s);" % host_ids.map { "?" }.join(", "), [status] + host_ids).map { |row| row[0] }
|
87
|
+
}
|
85
88
|
tags ||= @options[:tags]
|
86
89
|
update_db
|
87
90
|
if host_ids.empty?
|
@@ -123,7 +126,7 @@ module Hotdog
|
|
123
126
|
host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
|
124
127
|
q = "SELECT DISTINCT tags.name FROM hosts_tags " \
|
125
128
|
"INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
|
126
|
-
|
129
|
+
"WHERE hosts_tags.host_id IN (%s) ORDER BY hosts_tags.host_id;" % host_ids.map { "?" }.join(", ")
|
127
130
|
execute(q, host_ids).map { |row| row.first }
|
128
131
|
}.uniq
|
129
132
|
end
|
@@ -142,11 +145,12 @@ module Hotdog
|
|
142
145
|
|
143
146
|
def get_host_fields(host_id, fields, options={})
|
144
147
|
field_values = {}
|
145
|
-
fields.uniq.each_slice(SQLITE_LIMIT_COMPOUND_SELECT -
|
148
|
+
fields.uniq.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 2).each do |fields|
|
146
149
|
q = "SELECT LOWER(tags.name), GROUP_CONCAT(tags.value, ',') FROM hosts_tags " \
|
147
150
|
"INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
|
148
|
-
|
151
|
+
"WHERE hosts_tags.host_id = ? AND tags.name IN (%s) " \
|
149
152
|
"GROUP BY tags.name;" % fields.map { "?" }.join(", ")
|
153
|
+
|
150
154
|
execute(q, [host_id] + fields).each do |row|
|
151
155
|
field_values[row[0]] = row[1]
|
152
156
|
end
|
@@ -162,11 +166,11 @@ module Hotdog
|
|
162
166
|
def get_hosts_field(host_ids, field, options={})
|
163
167
|
host_ids = Array(host_ids)
|
164
168
|
if /\Ahost\z/i =~ field
|
165
|
-
result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
|
169
|
+
result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 1).flat_map { |host_ids|
|
166
170
|
execute("SELECT name FROM hosts WHERE id IN (%s) ORDER BY id;" % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.to_a }
|
167
171
|
}
|
168
172
|
else
|
169
|
-
result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT -
|
173
|
+
result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 2).flat_map { |host_ids|
|
170
174
|
q = "SELECT LOWER(tags.name), GROUP_CONCAT(tags.value, ',') FROM hosts_tags " \
|
171
175
|
"INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
|
172
176
|
"WHERE hosts_tags.host_id IN (%s) AND tags.name = ? " \
|
@@ -229,7 +233,7 @@ module Hotdog
|
|
229
233
|
def __open_db(options={})
|
230
234
|
begin
|
231
235
|
db = SQLite3::Database.new(persistent_db_path)
|
232
|
-
db.execute("SELECT hosts_tags.host_id FROM hosts_tags INNER JOIN hosts ON hosts_tags.host_id = hosts.id INNER JOIN tags ON hosts_tags.tag_id = tags.id LIMIT 1;")
|
236
|
+
db.execute("SELECT hosts_tags.host_id, hosts.source, hosts.status FROM hosts_tags INNER JOIN hosts ON hosts_tags.host_id = hosts.id INNER JOIN tags ON hosts_tags.tag_id = tags.id LIMIT 1;")
|
233
237
|
db
|
234
238
|
rescue SQLite3::BusyException # database is locked
|
235
239
|
sleep(rand)
|
@@ -261,9 +265,25 @@ module Hotdog
|
|
261
265
|
|
262
266
|
def create_db(db, options={})
|
263
267
|
options = @options.merge(options)
|
264
|
-
|
268
|
+
requests = {all_downtimes: "/api/v1/downtime", all_tags: "/api/v1/tags/hosts"}
|
269
|
+
begin
|
270
|
+
parallelism = Parallel.processor_count
|
271
|
+
# generate payload before forking threads to avoid fetching keys multiple times
|
272
|
+
query = URI.encode_www_form(api_key: application.api_key, application_key: application.application_key)
|
273
|
+
responses = Hash[Parallel.map(requests, in_threads: parallelism) { |name, request_path|
|
274
|
+
[name, datadog_get(request_path, query)]
|
275
|
+
}]
|
276
|
+
rescue => error
|
277
|
+
STDERR.puts(error.message)
|
278
|
+
exit(1)
|
279
|
+
end
|
280
|
+
all_tags = prepare_tags(responses.fetch(:all_tags, {}))
|
281
|
+
all_downtimes = prepare_downtimes(responses.fetch(:all_downtimes, {}))
|
282
|
+
if not all_downtimes.empty?
|
283
|
+
logger.info("ignore host(s) with scheduled downtimes: #{all_downtimes.inspect}")
|
284
|
+
end
|
265
285
|
db.transaction do
|
266
|
-
execute_db(db, "CREATE TABLE IF NOT EXISTS hosts (id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255) NOT NULL COLLATE NOCASE);")
|
286
|
+
execute_db(db, "CREATE TABLE IF NOT EXISTS hosts (id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255) NOT NULL COLLATE NOCASE, source INTEGER NOT NULL DEFAULT #{SOURCE_DATADOG}, status INTEGER NOT NULL DEFAULT #{STATUS_PENDING});")
|
267
287
|
execute_db(db, "CREATE UNIQUE INDEX IF NOT EXISTS hosts_name ON hosts (name);")
|
268
288
|
execute_db(db, "CREATE TABLE IF NOT EXISTS tags (id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(200) NOT NULL COLLATE NOCASE, value VARCHAR(200) NOT NULL COLLATE NOCASE);")
|
269
289
|
execute_db(db, "CREATE UNIQUE INDEX IF NOT EXISTS tags_name_value ON tags (name, value);")
|
@@ -274,7 +294,7 @@ module Hotdog
|
|
274
294
|
create_tags(db, known_tags)
|
275
295
|
|
276
296
|
known_hosts = all_tags.values.reduce(:+).uniq
|
277
|
-
create_hosts(db, known_hosts)
|
297
|
+
create_hosts(db, known_hosts, all_downtimes)
|
278
298
|
|
279
299
|
all_tags.each do |tag, hosts|
|
280
300
|
associate_tag_hosts(db, tag, hosts)
|
@@ -307,44 +327,42 @@ module Hotdog
|
|
307
327
|
end
|
308
328
|
end
|
309
329
|
|
310
|
-
def
|
330
|
+
def datadog_get(request_path, query=nil)
|
331
|
+
# TODO: make this pluggable
|
311
332
|
endpoint = options[:endpoint]
|
312
|
-
|
313
|
-
|
333
|
+
query ||= URI.encode_www_form(api_key: application.api_key, application_key: application.application_key)
|
334
|
+
uri = URI.join(endpoint, "#{request_path}?#{query}")
|
314
335
|
begin
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
[name, MultiJson.load(response)]
|
321
|
-
rescue OpenURI::HTTPError => error
|
322
|
-
code, _body = error.io.status
|
323
|
-
raise(RuntimeError.new("dog.get_#{name}() returns [#{code.inspect}, ...]"))
|
324
|
-
end
|
325
|
-
}]
|
326
|
-
rescue => error
|
327
|
-
STDERR.puts(error.message)
|
328
|
-
exit(1)
|
336
|
+
response = uri.open("User-Agent" => "hotdog/#{Hotdog::VERSION}") { |fp| fp.read }
|
337
|
+
MultiJson.load(response)
|
338
|
+
rescue OpenURI::HTTPError => error
|
339
|
+
code, _body = error.io.status
|
340
|
+
raise(RuntimeError.new("dog.get_#{name}() returns [#{code.inspect}, ...]"))
|
329
341
|
end
|
342
|
+
end
|
343
|
+
|
344
|
+
def prepare_tags(tags)
|
345
|
+
Hash(tags).fetch("tags", {})
|
346
|
+
end
|
347
|
+
|
348
|
+
def prepare_downtimes(downtimes)
|
330
349
|
now = Time.new.to_i
|
331
|
-
downtimes
|
350
|
+
Array(downtimes).select { |downtime|
|
332
351
|
# active downtimes
|
333
352
|
downtime["active"] and ( downtime["start"].nil? or downtime["start"] < now ) and ( downtime["end"].nil? or now <= downtime["end"] ) and downtime["monitor_id"].nil?
|
334
353
|
}.flat_map { |downtime|
|
335
354
|
# find host scopes
|
336
355
|
downtime["scope"].select { |scope| scope.start_with?("host:") }.map { |scope| scope.sub(/\Ahost:/, "") }
|
337
356
|
}
|
338
|
-
if not downtimes.empty?
|
339
|
-
logger.info("ignore host(s) with scheduled downtimes: #{downtimes.inspect}")
|
340
|
-
end
|
341
|
-
Hash[responses.fetch(:all_tags, {}).fetch("tags", []).map { |tag, hosts| [tag, hosts.reject { |host| downtimes.include?(host) }] }]
|
342
357
|
end
|
343
358
|
|
344
|
-
def create_hosts(db, hosts)
|
345
|
-
hosts.each_slice(SQLITE_LIMIT_COMPOUND_SELECT) do |hosts|
|
346
|
-
q = "INSERT OR IGNORE INTO hosts (name) VALUES %s" % hosts.map { "(?)" }.join(", ")
|
347
|
-
execute_db(db, q, hosts
|
359
|
+
def create_hosts(db, hosts, downtimes)
|
360
|
+
hosts.each_slice(SQLITE_LIMIT_COMPOUND_SELECT / 2) do |hosts|
|
361
|
+
q = "INSERT OR IGNORE INTO hosts (name, status) VALUES %s;" % hosts.map { "(?, ?)" }.join(", ")
|
362
|
+
execute_db(db, q, hosts.map { |host|
|
363
|
+
status = downtimes.include?(host) ? STATUS_STOPPED : STATUS_RUNNING
|
364
|
+
[host, status]
|
365
|
+
})
|
348
366
|
end
|
349
367
|
# create virtual `host` tag
|
350
368
|
execute_db(db, "INSERT OR IGNORE INTO tags (name, value) SELECT 'host', hosts.name FROM hosts;")
|