hotdog 0.26.0 → 0.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 5d13acaecbe1b4f13f6b9956f2e75cf69cfaff23
4
- data.tar.gz: acdef611442f0a968b0b9810c962b25864c0b406
3
+ metadata.gz: ed4a83ac54e12aecba688f7f0bb7b563f9cee80c
4
+ data.tar.gz: '038bbcb0dbc3e5e541c9a433ec64946beec79b62'
5
5
  SHA512:
6
- metadata.gz: a6f4cc4380eea63cee183a0baff9b606e40e94cdc6522e5e16e33e44acb161987d6001ebf743fa6a7f054c4ed8cc034edd8494893a4c9736be58376efbd2ad7d
7
- data.tar.gz: 950a18e1686c18591e474900353f738c78cc5f568c830dd41f43b231905eebdfe3bc415fd1662d710e9ec1072e4bba3ab050886f239a20aa35e8c237f7db75e0
6
+ metadata.gz: 35816f0033e867223cd699e3740566339896d6bf53e08f5e140c245e6e4b23abb7eebbaf996ff69c096e096ff1f9b2acab24b72700198c4063cbd911adffbbc9
7
+ data.tar.gz: d4158608de22ab5bee02e542bedf87f5e1fe2b2dac6b5bb82e02529930763830c0c371789ad1cd3fdec2adb634c0ad1c7913d427c57287d012ad0d07b2580421
@@ -11,11 +11,32 @@ require "hotdog/version"
11
11
  module Hotdog
12
12
  SQLITE_LIMIT_COMPOUND_SELECT = 500 # TODO: get actual value from `sqlite3_limit()`?
13
13
 
14
+ # only datadog is supported as of Sep 5, 2017
15
+ SOURCE_DATADOG = 0x01
16
+
17
+ # | status | description |
18
+ # | -------- | ------------- |
19
+ # | 00000000 | pending |
20
+ # | 00010000 | running |
21
+ # | 00100000 | shutting-down |
22
+ # | 00110000 | terminated |
23
+ # | 01000000 | stopping |
24
+ # | 01010000 | stopped |
25
+ STATUS_PENDING = 0b00000000
26
+ STATUS_RUNNING = 0b00010000
27
+ STATUS_SHUTTING_DOWN = 0b00100000
28
+ STATUS_TERMINATED = 0b00110000
29
+ STATUS_STOPPING = 0b01000000
30
+ STATUS_STOPPED = 0b01010000
31
+
32
+ VERBOSITY_NULL = 0
33
+ VERBOSITY_INFO = 1
34
+ VERBOSITY_DEBUG = 2
35
+ VERBOSITY_TRACE = 4
36
+
14
37
  class Application
15
38
  def initialize()
16
- @logger = Logger.new(STDERR).tap { |logger|
17
- logger.level = Logger::INFO
18
- }
39
+ @logger = Logger.new(STDERR)
19
40
  @optparse = OptionParser.new
20
41
  @optparse.version = Hotdog::VERSION
21
42
  @options = {
@@ -30,6 +51,7 @@ module Hotdog
30
51
  force: false,
31
52
  format: "text",
32
53
  headers: false,
54
+ status: STATUS_RUNNING,
33
55
  listing: false,
34
56
  logger: @logger,
35
57
  max_time: 5,
@@ -41,7 +63,9 @@ module Hotdog
41
63
  tags: [],
42
64
  display_search_tags: false,
43
65
  verbose: false,
66
+ verbosity: VERBOSITY_NULL,
44
67
  }
68
+
45
69
  define_options
46
70
  end
47
71
  attr_reader :logger
@@ -85,10 +109,18 @@ module Hotdog
85
109
 
86
110
  options[:formatter] = get_formatter(options[:format])
87
111
 
88
- if options[:debug] or options[:verbose]
112
+ if ( options[:debug] or options[:verbose] ) and ( options[:verbosity] < VERBOSITY_DEBUG )
113
+ options[:verbosity] = VERBOSITY_DEBUG
114
+ end
115
+
116
+ if VERBOSITY_DEBUG <= options[:verbosity]
89
117
  options[:logger].level = Logger::DEBUG
90
118
  else
91
- options[:logger].level = Logger::INFO
119
+ if VERBOSITY_INFO <= options[:verbosity]
120
+ options[:logger].level = Logger::INFO
121
+ else
122
+ options[:logger].level = Logger::WARN
123
+ end
92
124
  end
93
125
 
94
126
  command.run(args, @options)
@@ -127,6 +159,10 @@ module Hotdog
127
159
  end
128
160
  end
129
161
 
162
+ def status()
163
+ options.fetch(:status, STATUS_RUNNING)
164
+ end
165
+
130
166
  private
131
167
  def define_options
132
168
  @optparse.on("--endpoint ENDPOINT", "Datadog API endpoint") do |endpoint|
@@ -168,6 +204,24 @@ module Hotdog
168
204
  @optparse.on("-h", "--[no-]headers", "Display headeres for each columns") do |v|
169
205
  options[:headers] = v
170
206
  end
207
+ @optparse.on("--status=STATUS", "Specify custom host status") do |v|
208
+ case v
209
+ when /\Apending\z/i
210
+ options[:status] = STATUS_PENDING
211
+ when /\Arunning\z/i
212
+ options[:status] = STATUS_RUNNING
213
+ when /\Ashutting-down\z/i
214
+ options[:status] = STATUS_SHUTTING_DOWN
215
+ when /\Aterminated\z/i
216
+ options[:status] = STATUS_TERMINATED
217
+ when /\Astopping\z/i
218
+ options[:status] = STATUS_STOPPING
219
+ when /\Astopped\z/i
220
+ options[:status] = STATUS_STOPPED
221
+ else
222
+ options[:status] = v.to_i
223
+ end
224
+ end
171
225
  @optparse.on("-l", "--[no-]listing", "Use listing format") do |v|
172
226
  options[:listing] = v
173
227
  end
@@ -180,8 +234,8 @@ module Hotdog
180
234
  @optparse.on("-x", "--display-search-tags", "Show tags used in search expression") do |v|
181
235
  options[:display_search_tags] = v
182
236
  end
183
- @optparse.on("-V", "--[no-]verbose", "Enable verbose mode") do |v|
184
- options[:verbose] = v
237
+ @optparse.on("-V", "-v", "--[no-]verbose", "Enable verbose mode") do |v|
238
+ options[:verbosity] += 1
185
239
  end
186
240
  @optparse.on("--[no-]offline", "Enable offline mode") do |v|
187
241
  options[:offline] = v
@@ -40,9 +40,9 @@ module Hotdog
40
40
  if open_db
41
41
  @db.transaction do
42
42
  sqlite_limit_compound_select = options[:sqlite_limit_compound_select] || SQLITE_LIMIT_COMPOUND_SELECT
43
- hosts.each_slice(sqlite_limit_compound_select) do |hosts|
44
- execute_db(@db, "DELETE FROM hosts_tags WHERE host_id IN ( SELECT id FROM hosts WHERE name IN (%s) );" % hosts.map { "?" }.join(", "), hosts)
45
- execute_db(@db, "DELETE FROM hosts WHERE name IN (%s);" % hosts.map { "?" }.join(", "), hosts)
43
+ hosts.each_slice(sqlite_limit_compound_select - 1) do |hosts|
44
+ q = "UPDATE hosts SET status = ? WHERE name IN (%s);" % hosts.map { "?" }.join(", ")
45
+ execute_db(@db, q, [STATUS_STOPPING] + hosts)
46
46
  end
47
47
  end
48
48
  end
@@ -105,7 +105,15 @@ module Hotdog
105
105
  JSON.pretty_generate(optimized.dump)
106
106
  }
107
107
  end
108
- optimized.evaluate(environment)
108
+ result = optimized.evaluate(environment)
109
+ if result.empty? and !$did_reload
110
+ $did_reload = true
111
+ environment.logger.info("reloading all hosts and tags.")
112
+ environment.reload(force: true)
113
+ optimized.evaluate(environment)
114
+ else
115
+ result
116
+ end
109
117
  else
110
118
  raise("parser error: unknown expression: #{node.inspect}")
111
119
  end
@@ -40,9 +40,6 @@ module Hotdog
40
40
  optparse.on("-u SSH_USER", "SSH login user name") do |user|
41
41
  options[:ssh_options]["User"] = user
42
42
  end
43
- optparse.on("-v", "--verbose", "Enable verbose ode") do |v|
44
- options[:verbose] = v
45
- end
46
43
  optparse.on("--filter=COMMAND", "Command to filter search result.") do |command|
47
44
  options[:filter_command] = command
48
45
  end
@@ -71,6 +68,7 @@ module Hotdog
71
68
  tuples, fields = get_hosts_with_search_tags(result0, node)
72
69
  tuples = filter_hosts(tuples)
73
70
  validate_hosts!(tuples, fields)
71
+ logger.info("target host(s): #{tuples.map {|tuple| tuple.first }.inspect}")
74
72
  run_main(tuples.map {|tuple| tuple.first }, options)
75
73
  end
76
74
 
@@ -125,7 +123,7 @@ module Hotdog
125
123
  cmdline << "-F" << File.expand_path(options[:ssh_config])
126
124
  end
127
125
  cmdline += options[:ssh_options].flat_map { |k, v| ["-o", "#{k}=#{v}"] }
128
- if options[:verbose]
126
+ if VERBOSITY_TRACE <= options[:verbosity]
129
127
  cmdline << "-v"
130
128
  end
131
129
  cmdline
@@ -168,12 +166,14 @@ module Hotdog
168
166
  i = 0
169
167
  each_readable([cmderr, cmdout]) do |readable|
170
168
  raw = readable.readline
171
- output_lock.synchronize do
172
- if readable == cmdout
173
- STDOUT.puts(prettify_output(raw, i, color, identifier))
174
- i += 1
175
- else
176
- STDERR.puts(raw)
169
+ if output
170
+ output_lock.synchronize do
171
+ if readable == cmdout
172
+ STDOUT.puts(prettify_output(raw, i, color, identifier))
173
+ i += 1
174
+ else
175
+ STDERR.puts(prettify_output(raw, nil, nil, identifier))
176
+ end
177
177
  end
178
178
  end
179
179
  end
@@ -213,8 +213,10 @@ module Hotdog
213
213
  end
214
214
  buf << identifier
215
215
  buf << ":"
216
- buf << i.to_s
217
- buf << ":"
216
+ if i
217
+ buf << i.to_s
218
+ buf << ":"
219
+ end
218
220
  if color
219
221
  buf << "\e[0m"
220
222
  end
@@ -81,7 +81,10 @@ module Hotdog
81
81
  end
82
82
 
83
83
  def get_hosts(host_ids, tags=nil)
84
- host_ids = Array(host_ids)
84
+ status = application.status || STATUS_RUNNING
85
+ host_ids = Array(host_ids).each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
86
+ execute("SELECT id FROM hosts WHERE status = ? AND id IN (%s);" % host_ids.map { "?" }.join(", "), [status] + host_ids).map { |row| row[0] }
87
+ }
85
88
  tags ||= @options[:tags]
86
89
  update_db
87
90
  if host_ids.empty?
@@ -123,7 +126,7 @@ module Hotdog
123
126
  host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
124
127
  q = "SELECT DISTINCT tags.name FROM hosts_tags " \
125
128
  "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
126
- "WHERE hosts_tags.host_id IN (%s) ORDER BY hosts_tags.host_id;" % host_ids.map { "?" }.join(", ")
129
+ "WHERE hosts_tags.host_id IN (%s) ORDER BY hosts_tags.host_id;" % host_ids.map { "?" }.join(", ")
127
130
  execute(q, host_ids).map { |row| row.first }
128
131
  }.uniq
129
132
  end
@@ -142,11 +145,12 @@ module Hotdog
142
145
 
143
146
  def get_host_fields(host_id, fields, options={})
144
147
  field_values = {}
145
- fields.uniq.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 1).each do |fields|
148
+ fields.uniq.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 2).each do |fields|
146
149
  q = "SELECT LOWER(tags.name), GROUP_CONCAT(tags.value, ',') FROM hosts_tags " \
147
150
  "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
148
- "WHERE hosts_tags.host_id = ? AND tags.name IN (%s) " \
151
+ "WHERE hosts_tags.host_id = ? AND tags.name IN (%s) " \
149
152
  "GROUP BY tags.name;" % fields.map { "?" }.join(", ")
153
+
150
154
  execute(q, [host_id] + fields).each do |row|
151
155
  field_values[row[0]] = row[1]
152
156
  end
@@ -162,11 +166,11 @@ module Hotdog
162
166
  def get_hosts_field(host_ids, field, options={})
163
167
  host_ids = Array(host_ids)
164
168
  if /\Ahost\z/i =~ field
165
- result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
169
+ result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 1).flat_map { |host_ids|
166
170
  execute("SELECT name FROM hosts WHERE id IN (%s) ORDER BY id;" % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.to_a }
167
171
  }
168
172
  else
169
- result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 1).flat_map { |host_ids|
173
+ result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 2).flat_map { |host_ids|
170
174
  q = "SELECT LOWER(tags.name), GROUP_CONCAT(tags.value, ',') FROM hosts_tags " \
171
175
  "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
172
176
  "WHERE hosts_tags.host_id IN (%s) AND tags.name = ? " \
@@ -229,7 +233,7 @@ module Hotdog
229
233
  def __open_db(options={})
230
234
  begin
231
235
  db = SQLite3::Database.new(persistent_db_path)
232
- db.execute("SELECT hosts_tags.host_id FROM hosts_tags INNER JOIN hosts ON hosts_tags.host_id = hosts.id INNER JOIN tags ON hosts_tags.tag_id = tags.id LIMIT 1;")
236
+ db.execute("SELECT hosts_tags.host_id, hosts.source, hosts.status FROM hosts_tags INNER JOIN hosts ON hosts_tags.host_id = hosts.id INNER JOIN tags ON hosts_tags.tag_id = tags.id LIMIT 1;")
233
237
  db
234
238
  rescue SQLite3::BusyException # database is locked
235
239
  sleep(rand)
@@ -261,9 +265,25 @@ module Hotdog
261
265
 
262
266
  def create_db(db, options={})
263
267
  options = @options.merge(options)
264
- all_tags = get_all_tags()
268
+ requests = {all_downtimes: "/api/v1/downtime", all_tags: "/api/v1/tags/hosts"}
269
+ begin
270
+ parallelism = Parallel.processor_count
271
+ # generate payload before forking threads to avoid fetching keys multiple times
272
+ query = URI.encode_www_form(api_key: application.api_key, application_key: application.application_key)
273
+ responses = Hash[Parallel.map(requests, in_threads: parallelism) { |name, request_path|
274
+ [name, datadog_get(request_path, query)]
275
+ }]
276
+ rescue => error
277
+ STDERR.puts(error.message)
278
+ exit(1)
279
+ end
280
+ all_tags = prepare_tags(responses.fetch(:all_tags, {}))
281
+ all_downtimes = prepare_downtimes(responses.fetch(:all_downtimes, {}))
282
+ if not all_downtimes.empty?
283
+ logger.info("ignore host(s) with scheduled downtimes: #{all_downtimes.inspect}")
284
+ end
265
285
  db.transaction do
266
- execute_db(db, "CREATE TABLE IF NOT EXISTS hosts (id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255) NOT NULL COLLATE NOCASE);")
286
+ execute_db(db, "CREATE TABLE IF NOT EXISTS hosts (id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255) NOT NULL COLLATE NOCASE, source INTEGER NOT NULL DEFAULT #{SOURCE_DATADOG}, status INTEGER NOT NULL DEFAULT #{STATUS_PENDING});")
267
287
  execute_db(db, "CREATE UNIQUE INDEX IF NOT EXISTS hosts_name ON hosts (name);")
268
288
  execute_db(db, "CREATE TABLE IF NOT EXISTS tags (id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(200) NOT NULL COLLATE NOCASE, value VARCHAR(200) NOT NULL COLLATE NOCASE);")
269
289
  execute_db(db, "CREATE UNIQUE INDEX IF NOT EXISTS tags_name_value ON tags (name, value);")
@@ -274,7 +294,7 @@ module Hotdog
274
294
  create_tags(db, known_tags)
275
295
 
276
296
  known_hosts = all_tags.values.reduce(:+).uniq
277
- create_hosts(db, known_hosts)
297
+ create_hosts(db, known_hosts, all_downtimes)
278
298
 
279
299
  all_tags.each do |tag, hosts|
280
300
  associate_tag_hosts(db, tag, hosts)
@@ -307,44 +327,42 @@ module Hotdog
307
327
  end
308
328
  end
309
329
 
310
- def get_all_tags() #==> Hash<Tag,Array<Host>>
330
+ def datadog_get(request_path, query=nil)
331
+ # TODO: make this pluggable
311
332
  endpoint = options[:endpoint]
312
- requests = {all_downtime: "/api/v1/downtime", all_tags: "/api/v1/tags/hosts"}
313
- query = URI.encode_www_form(api_key: application.api_key, application_key: application.application_key)
333
+ query ||= URI.encode_www_form(api_key: application.api_key, application_key: application.application_key)
334
+ uri = URI.join(endpoint, "#{request_path}?#{query}")
314
335
  begin
315
- parallelism = Parallel.processor_count
316
- responses = Hash[Parallel.map(requests, in_threads: parallelism) { |name, request_path|
317
- uri = URI.join(endpoint, "#{request_path}?#{query}")
318
- begin
319
- response = uri.open("User-Agent" => "hotdog/#{Hotdog::VERSION}") { |fp| fp.read }
320
- [name, MultiJson.load(response)]
321
- rescue OpenURI::HTTPError => error
322
- code, _body = error.io.status
323
- raise(RuntimeError.new("dog.get_#{name}() returns [#{code.inspect}, ...]"))
324
- end
325
- }]
326
- rescue => error
327
- STDERR.puts(error.message)
328
- exit(1)
336
+ response = uri.open("User-Agent" => "hotdog/#{Hotdog::VERSION}") { |fp| fp.read }
337
+ MultiJson.load(response)
338
+ rescue OpenURI::HTTPError => error
339
+ code, _body = error.io.status
340
+ raise(RuntimeError.new("dog.get_#{name}() returns [#{code.inspect}, ...]"))
329
341
  end
342
+ end
343
+
344
+ def prepare_tags(tags)
345
+ Hash(tags).fetch("tags", {})
346
+ end
347
+
348
+ def prepare_downtimes(downtimes)
330
349
  now = Time.new.to_i
331
- downtimes = responses.fetch(:all_downtime, []).select { |downtime|
350
+ Array(downtimes).select { |downtime|
332
351
  # active downtimes
333
352
  downtime["active"] and ( downtime["start"].nil? or downtime["start"] < now ) and ( downtime["end"].nil? or now <= downtime["end"] ) and downtime["monitor_id"].nil?
334
353
  }.flat_map { |downtime|
335
354
  # find host scopes
336
355
  downtime["scope"].select { |scope| scope.start_with?("host:") }.map { |scope| scope.sub(/\Ahost:/, "") }
337
356
  }
338
- if not downtimes.empty?
339
- logger.info("ignore host(s) with scheduled downtimes: #{downtimes.inspect}")
340
- end
341
- Hash[responses.fetch(:all_tags, {}).fetch("tags", []).map { |tag, hosts| [tag, hosts.reject { |host| downtimes.include?(host) }] }]
342
357
  end
343
358
 
344
- def create_hosts(db, hosts)
345
- hosts.each_slice(SQLITE_LIMIT_COMPOUND_SELECT) do |hosts|
346
- q = "INSERT OR IGNORE INTO hosts (name) VALUES %s" % hosts.map { "(?)" }.join(", ")
347
- execute_db(db, q, hosts)
359
+ def create_hosts(db, hosts, downtimes)
360
+ hosts.each_slice(SQLITE_LIMIT_COMPOUND_SELECT / 2) do |hosts|
361
+ q = "INSERT OR IGNORE INTO hosts (name, status) VALUES %s;" % hosts.map { "(?, ?)" }.join(", ")
362
+ execute_db(db, q, hosts.map { |host|
363
+ status = downtimes.include?(host) ? STATUS_STOPPED : STATUS_RUNNING
364
+ [host, status]
365
+ })
348
366
  end
349
367
  # create virtual `host` tag
350
368
  execute_db(db, "INSERT OR IGNORE INTO tags (name, value) SELECT 'host', hosts.name FROM hosts;")