hotdog 0.26.0 → 0.27.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5d13acaecbe1b4f13f6b9956f2e75cf69cfaff23
-  data.tar.gz: acdef611442f0a968b0b9810c962b25864c0b406
+  metadata.gz: ed4a83ac54e12aecba688f7f0bb7b563f9cee80c
+  data.tar.gz: '038bbcb0dbc3e5e541c9a433ec64946beec79b62'
 SHA512:
-  metadata.gz: a6f4cc4380eea63cee183a0baff9b606e40e94cdc6522e5e16e33e44acb161987d6001ebf743fa6a7f054c4ed8cc034edd8494893a4c9736be58376efbd2ad7d
-  data.tar.gz: 950a18e1686c18591e474900353f738c78cc5f568c830dd41f43b231905eebdfe3bc415fd1662d710e9ec1072e4bba3ab050886f239a20aa35e8c237f7db75e0
+  metadata.gz: 35816f0033e867223cd699e3740566339896d6bf53e08f5e140c245e6e4b23abb7eebbaf996ff69c096e096ff1f9b2acab24b72700198c4063cbd911adffbbc9
+  data.tar.gz: d4158608de22ab5bee02e542bedf87f5e1fe2b2dac6b5bb82e02529930763830c0c371789ad1cd3fdec2adb634c0ad1c7913d427c57287d012ad0d07b2580421

data/lib/hotdog/application.rb

@@ -11,11 +11,32 @@ require "hotdog/version"
 module Hotdog
   SQLITE_LIMIT_COMPOUND_SELECT = 500 # TODO: get actual value from `sqlite3_limit()`?
 
+  # only datadog is supported as of Sep 5, 2017
+  SOURCE_DATADOG = 0x01
+
+  # | status   | description   |
+  # | -------- | ------------- |
+  # | 00000000 | pending       |
+  # | 00010000 | running       |
+  # | 00100000 | shutting-down |
+  # | 00110000 | terminated    |
+  # | 01000000 | stopping      |
+  # | 01010000 | stopped       |
+  STATUS_PENDING       = 0b00000000
+  STATUS_RUNNING       = 0b00010000
+  STATUS_SHUTTING_DOWN = 0b00100000
+  STATUS_TERMINATED    = 0b00110000
+  STATUS_STOPPING      = 0b01000000
+  STATUS_STOPPED       = 0b01010000
+
+  VERBOSITY_NULL  = 0
+  VERBOSITY_INFO  = 1
+  VERBOSITY_DEBUG = 2
+  VERBOSITY_TRACE = 4
+
   class Application
     def initialize()
-      @logger = Logger.new(STDERR).tap { |logger|
-        logger.level = Logger::INFO
-      }
+      @logger = Logger.new(STDERR)
       @optparse = OptionParser.new
       @optparse.version = Hotdog::VERSION
       @options = {
@@ -30,6 +51,7 @@ module Hotdog
         force: false,
         format: "text",
         headers: false,
+        status: STATUS_RUNNING,
         listing: false,
         logger: @logger,
         max_time: 5,
@@ -41,7 +63,9 @@ module Hotdog
         tags: [],
         display_search_tags: false,
         verbose: false,
+        verbosity: VERBOSITY_NULL,
       }
+
       define_options
     end
     attr_reader :logger
@@ -85,10 +109,18 @@ module Hotdog
 
         options[:formatter] = get_formatter(options[:format])
 
-        if options[:debug] or options[:verbose]
+        if ( options[:debug] or options[:verbose] ) and ( options[:verbosity] < VERBOSITY_DEBUG )
+          options[:verbosity] = VERBOSITY_DEBUG
+        end
+
+        if VERBOSITY_DEBUG <= options[:verbosity]
           options[:logger].level = Logger::DEBUG
         else
-          options[:logger].level = Logger::INFO
+          if VERBOSITY_INFO <= options[:verbosity]
+            options[:logger].level = Logger::INFO
+          else
+            options[:logger].level = Logger::WARN
+          end
         end
 
         command.run(args, @options)
@@ -127,6 +159,10 @@ module Hotdog
       end
     end
 
+    def status()
+      options.fetch(:status, STATUS_RUNNING)
+    end
+
     private
     def define_options
       @optparse.on("--endpoint ENDPOINT", "Datadog API endpoint") do |endpoint|
@@ -168,6 +204,24 @@ module Hotdog
       @optparse.on("-h", "--[no-]headers", "Display headeres for each columns") do |v|
         options[:headers] = v
       end
+      @optparse.on("--status=STATUS", "Specify custom host status") do |v|
+        case v
+        when /\Apending\z/i
+          options[:status] = STATUS_PENDING
+        when /\Arunning\z/i
+          options[:status] = STATUS_RUNNING
+        when /\Ashutting-down\z/i
+          options[:status] = STATUS_SHUTTING_DOWN
+        when /\Aterminated\z/i
+          options[:status] = STATUS_TERMINATED
+        when /\Astopping\z/i
+          options[:status] = STATUS_STOPPING
+        when /\Astopped\z/i
+          options[:status] = STATUS_STOPPED
+        else
+          options[:status] = v.to_i
+        end
+      end
       @optparse.on("-l", "--[no-]listing", "Use listing format") do |v|
         options[:listing] = v
       end
@@ -180,8 +234,8 @@ module Hotdog
       @optparse.on("-x", "--display-search-tags", "Show tags used in search expression") do |v|
         options[:display_search_tags] = v
       end
-      @optparse.on("-V", "--[no-]verbose", "Enable verbose mode") do |v|
-        options[:verbose] = v
+      @optparse.on("-V", "-v", "--[no-]verbose", "Enable verbose mode") do |v|
+        options[:verbosity] += 1
       end
       @optparse.on("--[no-]offline", "Enable offline mode") do |v|
         options[:offline] = v

data/lib/hotdog/commands/down.rb

@@ -40,9 +40,9 @@ module Hotdog
             if open_db
               @db.transaction do
                 sqlite_limit_compound_select = options[:sqlite_limit_compound_select] || SQLITE_LIMIT_COMPOUND_SELECT
-                hosts.each_slice(sqlite_limit_compound_select) do |hosts|
-                  execute_db(@db, "DELETE FROM hosts_tags WHERE host_id IN ( SELECT id FROM hosts WHERE name IN (%s) );" % hosts.map { "?" }.join(", "), hosts)
-                  execute_db(@db, "DELETE FROM hosts WHERE name IN (%s);" % hosts.map { "?" }.join(", "), hosts)
+                hosts.each_slice(sqlite_limit_compound_select - 1) do |hosts|
+                  q = "UPDATE hosts SET status = ? WHERE name IN (%s);" % hosts.map { "?" }.join(", ")
+                  execute_db(@db, q, [STATUS_STOPPING] + hosts)
                 end
               end
             end

data/lib/hotdog/commands/search.rb

@@ -105,7 +105,15 @@ module Hotdog
               JSON.pretty_generate(optimized.dump)
             }
           end
-          optimized.evaluate(environment)
+          result = optimized.evaluate(environment)
+          if result.empty? and !$did_reload
+            $did_reload = true
+            environment.logger.info("reloading all hosts and tags.")
+            environment.reload(force: true)
+            optimized.evaluate(environment)
+          else
+            result
+          end
         else
           raise("parser error: unknown expression: #{node.inspect}")
         end

data/lib/hotdog/commands/ssh.rb

@@ -40,9 +40,6 @@ module Hotdog
         optparse.on("-u SSH_USER", "SSH login user name") do |user|
           options[:ssh_options]["User"] = user
         end
-        optparse.on("-v", "--verbose", "Enable verbose ode") do |v|
-          options[:verbose] = v
-        end
         optparse.on("--filter=COMMAND", "Command to filter search result.") do |command|
           options[:filter_command] = command
         end
@@ -71,6 +68,7 @@ module Hotdog
         tuples, fields = get_hosts_with_search_tags(result0, node)
         tuples = filter_hosts(tuples)
         validate_hosts!(tuples, fields)
+        logger.info("target host(s): #{tuples.map {|tuple| tuple.first }.inspect}")
         run_main(tuples.map {|tuple| tuple.first }, options)
       end
 
@@ -125,7 +123,7 @@ module Hotdog
           cmdline << "-F" << File.expand_path(options[:ssh_config])
         end
         cmdline += options[:ssh_options].flat_map { |k, v| ["-o", "#{k}=#{v}"] }
-        if options[:verbose]
+        if VERBOSITY_TRACE <= options[:verbosity]
           cmdline << "-v"
         end
         cmdline
@@ -168,12 +166,14 @@ module Hotdog
           i = 0
           each_readable([cmderr, cmdout]) do |readable|
             raw = readable.readline
-            output_lock.synchronize do
-              if readable == cmdout
-                STDOUT.puts(prettify_output(raw, i, color, identifier))
-                i += 1
-              else
-                STDERR.puts(raw)
+            if output
+              output_lock.synchronize do
+                if readable == cmdout
+                  STDOUT.puts(prettify_output(raw, i, color, identifier))
+                  i += 1
+                else
+                  STDERR.puts(prettify_output(raw, nil, nil, identifier))
+                end
               end
             end
           end
@@ -213,8 +213,10 @@ module Hotdog
           end
           buf << identifier
           buf << ":"
-          buf << i.to_s
-          buf << ":"
+          if i
+            buf << i.to_s
+            buf << ":"
+          end
           if color
             buf << "\e[0m"
           end

data/lib/hotdog/commands.rb

@@ -81,7 +81,10 @@ module Hotdog
       end
 
       def get_hosts(host_ids, tags=nil)
-        host_ids = Array(host_ids)
+        status = application.status || STATUS_RUNNING
+        host_ids = Array(host_ids).each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
+          execute("SELECT id FROM hosts WHERE status = ? AND id IN (%s);" % host_ids.map { "?" }.join(", "), [status] + host_ids).map { |row| row[0] }
+        }
         tags ||= @options[:tags]
         update_db
         if host_ids.empty?
@@ -123,7 +126,7 @@ module Hotdog
         host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
           q = "SELECT DISTINCT tags.name FROM hosts_tags " \
                 "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
-                  "WHERE hosts_tags.host_id IN (%s) ORDER BY hosts_tags.host_id;" % host_ids.map { "?" }.join(", ")
+                "WHERE hosts_tags.host_id IN (%s) ORDER BY hosts_tags.host_id;" % host_ids.map { "?" }.join(", ")
           execute(q, host_ids).map { |row| row.first }
         }.uniq
       end
@@ -142,11 +145,12 @@ module Hotdog
 
       def get_host_fields(host_id, fields, options={})
         field_values = {}
-        fields.uniq.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 1).each do |fields|
+        fields.uniq.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 2).each do |fields|
           q = "SELECT LOWER(tags.name), GROUP_CONCAT(tags.value, ',') FROM hosts_tags " \
                 "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
-                  "WHERE hosts_tags.host_id = ? AND tags.name IN (%s) " \
+                "WHERE hosts_tags.host_id = ? AND tags.name IN (%s) " \
                     "GROUP BY tags.name;" % fields.map { "?" }.join(", ")
+
           execute(q, [host_id] + fields).each do |row|
             field_values[row[0]] = row[1]
           end
@@ -162,11 +166,11 @@ module Hotdog
       def get_hosts_field(host_ids, field, options={})
         host_ids = Array(host_ids)
         if /\Ahost\z/i =~ field
-          result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
+          result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 1).flat_map { |host_ids|
             execute("SELECT name FROM hosts WHERE id IN (%s) ORDER BY id;" % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.to_a }
           }
         else
-          result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 1).flat_map { |host_ids|
+          result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 2).flat_map { |host_ids|
             q = "SELECT LOWER(tags.name), GROUP_CONCAT(tags.value, ',') FROM hosts_tags " \
                   "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
                     "WHERE hosts_tags.host_id IN (%s) AND tags.name = ? " \
@@ -229,7 +233,7 @@ module Hotdog
       def __open_db(options={})
         begin
           db = SQLite3::Database.new(persistent_db_path)
-          db.execute("SELECT hosts_tags.host_id FROM hosts_tags INNER JOIN hosts ON hosts_tags.host_id = hosts.id INNER JOIN tags ON hosts_tags.tag_id = tags.id LIMIT 1;")
+          db.execute("SELECT hosts_tags.host_id, hosts.source, hosts.status FROM hosts_tags INNER JOIN hosts ON hosts_tags.host_id = hosts.id INNER JOIN tags ON hosts_tags.tag_id = tags.id LIMIT 1;")
           db
         rescue SQLite3::BusyException # database is locked
           sleep(rand)
@@ -261,9 +265,25 @@ module Hotdog
 
       def create_db(db, options={})
         options = @options.merge(options)
-        all_tags = get_all_tags()
+        requests = {all_downtimes: "/api/v1/downtime", all_tags: "/api/v1/tags/hosts"}
+        begin
+          parallelism = Parallel.processor_count
+          # generate payload before forking threads to avoid fetching keys multiple times
+          query = URI.encode_www_form(api_key: application.api_key, application_key: application.application_key)
+          responses = Hash[Parallel.map(requests, in_threads: parallelism) { |name, request_path|
+            [name, datadog_get(request_path, query)]
+          }]
+        rescue => error
+          STDERR.puts(error.message)
+          exit(1)
+        end
+        all_tags = prepare_tags(responses.fetch(:all_tags, {}))
+        all_downtimes = prepare_downtimes(responses.fetch(:all_downtimes, {}))
+        if not all_downtimes.empty?
+          logger.info("ignore host(s) with scheduled downtimes: #{all_downtimes.inspect}")
+        end
         db.transaction do
-          execute_db(db, "CREATE TABLE IF NOT EXISTS hosts (id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255) NOT NULL COLLATE NOCASE);")
+          execute_db(db, "CREATE TABLE IF NOT EXISTS hosts (id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255) NOT NULL COLLATE NOCASE, source INTEGER NOT NULL DEFAULT #{SOURCE_DATADOG}, status INTEGER NOT NULL DEFAULT #{STATUS_PENDING});")
           execute_db(db, "CREATE UNIQUE INDEX IF NOT EXISTS hosts_name ON hosts (name);")
           execute_db(db, "CREATE TABLE IF NOT EXISTS tags (id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(200) NOT NULL COLLATE NOCASE, value VARCHAR(200) NOT NULL COLLATE NOCASE);")
           execute_db(db, "CREATE UNIQUE INDEX IF NOT EXISTS tags_name_value ON tags (name, value);")
@@ -274,7 +294,7 @@ module Hotdog
           create_tags(db, known_tags)
 
           known_hosts = all_tags.values.reduce(:+).uniq
-          create_hosts(db, known_hosts)
+          create_hosts(db, known_hosts, all_downtimes)
 
           all_tags.each do |tag, hosts|
             associate_tag_hosts(db, tag, hosts)
@@ -307,44 +327,42 @@ module Hotdog
         end
       end
 
-      def get_all_tags() #==> Hash<Tag,Array<Host>>
+      def datadog_get(request_path, query=nil)
+        # TODO: make this pluggable
         endpoint = options[:endpoint]
-        requests = {all_downtime: "/api/v1/downtime", all_tags: "/api/v1/tags/hosts"}
-        query = URI.encode_www_form(api_key: application.api_key, application_key: application.application_key)
+        query ||= URI.encode_www_form(api_key: application.api_key, application_key: application.application_key)
+        uri = URI.join(endpoint, "#{request_path}?#{query}")
         begin
-          parallelism = Parallel.processor_count
-          responses = Hash[Parallel.map(requests, in_threads: parallelism) { |name, request_path|
-            uri = URI.join(endpoint, "#{request_path}?#{query}")
-            begin
-              response = uri.open("User-Agent" => "hotdog/#{Hotdog::VERSION}") { |fp| fp.read }
-              [name, MultiJson.load(response)]
-            rescue OpenURI::HTTPError => error
-              code, _body = error.io.status
-              raise(RuntimeError.new("dog.get_#{name}() returns [#{code.inspect}, ...]"))
-            end
-          }]
-        rescue => error
-          STDERR.puts(error.message)
-          exit(1)
+          response = uri.open("User-Agent" => "hotdog/#{Hotdog::VERSION}") { |fp| fp.read }
+          MultiJson.load(response)
+        rescue OpenURI::HTTPError => error
+          code, _body = error.io.status
+          raise(RuntimeError.new("dog.get_#{name}() returns [#{code.inspect}, ...]"))
         end
+      end
+
+      def prepare_tags(tags)
+        Hash(tags).fetch("tags", {})
+      end
+
+      def prepare_downtimes(downtimes)
         now = Time.new.to_i
-        downtimes = responses.fetch(:all_downtime, []).select { |downtime|
+        Array(downtimes).select { |downtime|
           # active downtimes
           downtime["active"] and ( downtime["start"].nil? or downtime["start"] < now ) and ( downtime["end"].nil? or now <= downtime["end"] ) and downtime["monitor_id"].nil?
         }.flat_map { |downtime|
           # find host scopes
           downtime["scope"].select { |scope| scope.start_with?("host:") }.map { |scope| scope.sub(/\Ahost:/, "") }
         }
-        if not downtimes.empty?
-          logger.info("ignore host(s) with scheduled downtimes: #{downtimes.inspect}")
-        end
-        Hash[responses.fetch(:all_tags, {}).fetch("tags", []).map { |tag, hosts| [tag, hosts.reject { |host| downtimes.include?(host) }] }]
       end
 
-      def create_hosts(db, hosts)
-        hosts.each_slice(SQLITE_LIMIT_COMPOUND_SELECT) do |hosts|
-          q = "INSERT OR IGNORE INTO hosts (name) VALUES %s" % hosts.map { "(?)" }.join(", ")
-          execute_db(db, q, hosts)
+      def create_hosts(db, hosts, downtimes)
+        hosts.each_slice(SQLITE_LIMIT_COMPOUND_SELECT / 2) do |hosts|
+          q = "INSERT OR IGNORE INTO hosts (name, status) VALUES %s;" % hosts.map { "(?, ?)" }.join(", ")
+          execute_db(db, q, hosts.map { |host|
+            status = downtimes.include?(host) ? STATUS_STOPPED : STATUS_RUNNING
+            [host, status]
+          })
         end
         # create virtual `host` tag
         execute_db(db, "INSERT OR IGNORE INTO tags (name, value) SELECT 'host', hosts.name FROM hosts;")