hotdog 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c0f33496f07a7069e8a6fab90cf456a98f80e986
-  data.tar.gz: d7bfd8130aa345ff212cabb30f06ddb83b286a46
+  metadata.gz: 9ec9c646216c6e398de74ef323b24368fb50ebb9
+  data.tar.gz: afbfbb8ae6fde51f3eb651385263c9294c22c864
 SHA512:
-  metadata.gz: 5c3aa62c94710e8bd3e1c738489782cfce401a3942d46be03128c4fc427719bba81f74d4cdf108c3cb9c3382ad83bdc2ba4085ca4f554ec2496fcdcb2b6a3b39
-  data.tar.gz: 19fce13c8757dd1fb0437abd99027d0d10ecbea68ca2db57808da119745fafbb0c704b588480f75682de5b7a12509829e2fe27c572ae1f09da5192ca5d2ace41
+  metadata.gz: c2ac8c3bb3d161d8f3ce102ce4a61c022d55b7dc165df776d5b8be31c85d3b103d88516bf60206efac7dbeabe7e26a4a2054d65a0728fff56fc5d2116a26dafc
+  data.tar.gz: 55df8e863c2b6234bd593073ea958fdb25d6232de94fd9f499ec6a6530abc5b1b26ac5f6b6f6e7237be16136743016b981a0854b2c6f77ac128aaa9039465627

data/lib/hotdog/application.rb

@@ -15,12 +15,13 @@ module Hotdog
       @options = {
         environment: "default",
         minimum_expiry: 28800, # 8 hours
-        random_expiry: 57600, # 16 hours
+        random_expiry: 604800, # 7 days
         force: false,
         formatter: get_formatter("plain").new,
         headers: false,
         listing: false,
         logger: Logger.new(STDERR),
+        max_time: 30,
         api_key: ENV["DATADOG_API_KEY"],
         application_key: ENV["DATADOG_APPLICATION_KEY"],
         print0: false,
@@ -54,14 +55,19 @@ module Hotdog
 
       begin
         command = ( args.shift || "help" )
-        run_command(command, args)
+        c = run_command(command, args)
+        if c.suspended?
+          exit(2)
+        end
       rescue Errno::EPIPE
         # nop
       end
     end
 
     def run_command(command, args=[])
-      get_command(command).new(@db, options.merge(application: self)).run(args)
+      get_command(command).new(@db, options.merge(application: self)).tap do |c|
+        c.run(args)
+      end
     end
 
     private
@@ -72,35 +78,38 @@ module Hotdog
       @optparse.on("--application-key APP_KEY", "Datadog application key") do |app_key|
         options[:application_key] = app_key
       end
-      @optparse.on("-0", "--null", "Use null character as separator") do
-        options[:print0] = true
+      @optparse.on("-0", "--null", "Use null character as separator") do |v|
+        options[:print0] = v
       end
-      @optparse.on("-1", "Use newline as separator") do
-        options[:print1] = true
+      @optparse.on("-1", "Use newline as separator") do |v|
+        options[:print1] = v
       end
-      @optparse.on("-d", "--debug", "Enable debug mode") do
-        options[:logger].level = Logger::DEBUG
+      @optparse.on("-d", "--[no-]debug", "Enable debug mode") do |v|
+        options[:logger].level = v ? Logger::DEBUG : Logger::INFO
       end
       @optparse.on("-E ENVIRONMENT", "--environment ENVIRONMENT", "Specify environment") do |environment|
         options[:environment] = environment
       end
-      @optparse.on("-f", "--force", "Enable force mode") do
-        options[:force] = true
+      @optparse.on("-f", "--[no-]force", "Enable force mode") do |v|
+        options[:force] = v
       end
       @optparse.on("-F FORMAT", "--format FORMAT", "Specify output format") do |format|
         options[:formatter] = get_formatter(format).new
       end
-      @optparse.on("-h", "--headers", "Display headeres for each columns") do |headers|
-        options[:headers] = headers
+      @optparse.on("-h", "--[no-]headers", "Display headeres for each columns") do |v|
+        options[:headers] = v
       end
-      @optparse.on("-l", "Use listing format") do
-        options[:listing] = true
+      @optparse.on("-l", "--[no-]listing", "Use listing format") do |v|
+        options[:listing] = v
       end
       @optparse.on("-a TAG", "-t TAG", "--tag TAG", "Use specified tag name/value") do |tag|
         options[:tags] += [tag]
       end
-      @optparse.on("-V", "--verbose", "Enable verbose mode") do |tag|
-        options[:logger].level = Logger::DEBUG
+      @optparse.on("-m SECONDS", "--max-time SECONDS", Integer, "Maximum time in seconds") do |seconds|
+        options[:max_time] = seconds
+      end
+      @optparse.on("-V", "--[no-]verbose", "Enable verbose mode") do |v|
+        options[:logger].level = v ? Logger::DEBUG : Logger::INFO
       end
     end
 

data/lib/hotdog/commands/init.rb

@@ -36,6 +36,9 @@ module Hotdog
         execute(<<-EOS)
           CREATE INDEX IF NOT EXISTS hosts_tags_expires_at ON hosts_tags ( expires_at );
         EOS
+        execute(<<-EOS)
+          CREATE INDEX IF NOT EXISTS hosts_tags_host_id_expires_at ON hosts_tags ( host_id, expires_at );
+        EOS
         application.run_command("update")
       end
     end

data/lib/hotdog/commands.rb

@@ -14,6 +14,8 @@ module Hotdog
         @application = options[:application]
         @options = options
         @dog = Dogapi::Client.new(options[:api_key], options[:application_key])
+        @started_at = Time.new
+        @suspended = false
       end
       attr_reader :application
       attr_reader :formatter
@@ -34,6 +36,10 @@ module Hotdog
         @db.execute(query, args)
       end
 
+      def suspended?()
+        @suspended
+      end
+
       private
       def format(result, options={})
         @formatter.format(result, @options.merge(options))
@@ -106,10 +112,12 @@ module Hotdog
       def update_hosts(options={})
         @db.transaction do
           if not options[:force]
+            # Update host list on every expirations to update frequently.
             @update_hosts_q1 ||= @db.prepare("SELECT MIN(expires_at) FROM hosts_tags;")
             logger.debug("update_hosts_q1()")
             if expires_at = @update_hosts_q1.execute().map { |row| row.first }.first
               if Time.new.to_i < expires_at
+                logger.debug("next update will run after %s." % [Time.at(expires_at)])
                 return
               else
                 logger.debug("minimum expires_at was %s. start updateing." % [Time.at(expires_at)])
@@ -120,26 +128,36 @@ module Hotdog
           end
 
           code, result = @dog.search("hosts:")
+          logger.debug("dog.serarch(%s) #==> [%s, %s]" % ["hosts:".inspect, code.inspect, result.inspect])
           if code.to_i / 100 != 2
             raise("HTTP #{code}: #{result.inspect}")
           end
 
+          execute(<<-EOS % result["results"]["hosts"].map { "LOWER(?)" }.join(", "), result["results"]["hosts"])
+            DELETE FROM hosts_tags WHERE host_id NOT IN
+              ( SELECT id FROM hosts WHERE LOWER(name) IN ( %s ) );
+          EOS
+
           result["results"]["hosts"].each do |host_name|
             @update_hosts_q2 ||= @db.prepare("INSERT OR IGNORE INTO hosts (name) VALUES (?);")
             logger.debug("update_hosts_q2(%s)" % [host_name.inspect])
             @update_hosts_q2.execute(host_name)
             update_host_tags(host_name, options)
-          end
 
-          execute(<<-EOS % result["results"]["hosts"].map { "LOWER(?)" }.join(", "), result["results"]["hosts"])
-            DELETE FROM hosts_tags WHERE host_id NOT IN
-              ( SELECT id FROM hosts WHERE LOWER(name) IN ( %s ) );
-          EOS
+            elapsed_time = Time.new - @started_at
+            if 0 < options[:max_time] and options[:max_time] < elapsed_time
+              logger.info("update host tags exceeded the maximum time (#{options[:max_time]} < #{elapsed_time}). will resume on next run.")
+              suspend_host_tags
+              break
+            end
+          end
         end
       end
 
       def update_tags(options={})
         @db.transaction do
+          resume_host_tags
+
           if options[:force]
             @update_tags_q1 ||= @db.prepare(<<-EOS)
               SELECT DISTINCT hosts_tags.host_id FROM hosts_tags;
@@ -154,12 +172,52 @@ module Hotdog
             EOS
             hosts = @update_tags_q2.execute(Time.new.to_i)
           end
-          hosts.each do |host_name|
-            update_host_tags(host_name, options)
+          hosts.each do |host_id|
+            @update_tags_q3 ||= @db.prepare("DELETE FROM hosts_tags WHERE host_id = ? AND hosts_tags.expires_at < ?;")
+            logger.debug("update_tags_q3(%s, %s)" % [host_id.inspect, Time.new.to_i])
+            @update_tags_q3.execute(host_id, Time.new.to_i)
+
+            update_host_tags(host_id, options)
+
+            elapsed_time = Time.new - @started_at
+            if 0 < options[:max_time] and options[:max_time] < elapsed_time
+              logger.info("update host tags exceeded the maximum time (#{options[:max_time]} < #{elapsed_time}). will resume on next run.")
+              suspend_host_tags
+              break
+            end
           end
         end
       end
 
+      # it'd be better to filter out this host/tag entry on displaying...
+      EMPTY_HOST_NAME = ""
+      EMPTY_TAG_NAME = ""
+      EMPTY_TAG_VALUE = ""
+      EMPTY_EXPIRES_AT = Time.at(0).to_i
+
+      def suspend_host_tags()
+        @suspended = true
+        @suspend_host_tags_q1 ||= @db.prepare("INSERT OR IGNORE INTO hosts (name) VALUES (?);")
+        @suspend_host_tags_q1.execute(EMPTY_HOST_NAME)
+        @suspend_host_tags_q2 ||= @db.prepare("INSERT OR IGNORE INTO tags (name, value) VALUES (?, ?);")
+        @suspend_host_tags_q2.execute(EMPTY_TAG_NAME, EMPTY_TAG_VALUE)
+        @suspend_host_tags_q3 ||= @db.prepare(<<-EOS)
+          INSERT OR REPLACE INTO hosts_tags (host_id, tag_id, expires_at)
+            SELECT host.id, tag.id, ? FROM
+              ( SELECT id FROM hosts WHERE name = ?) AS host,
+              ( SELECT id FROM tags WHERE name = ? AND value = ? ) AS tag;
+        EOS
+        @suspend_host_tags_q3.execute(EMPTY_EXPIRES_AT, EMPTY_HOST_NAME, EMPTY_TAG_NAME, EMPTY_TAG_VALUE)
+      end
+
+      def resume_host_tags()
+        @resume_host_tags_q1 ||= @db.prepare(<<-EOS)
+          DELETE FROM hosts_tags
+            WHERE host_id IN ( SELECT id FROM hosts WHERE name = ? ) AND tag_id IN ( SELECT id FROM tags WHERE name = ? AND value = ? );
+        EOS
+        @resume_host_tags_q1.execute(EMPTY_HOST_NAME, EMPTY_TAG_NAME, EMPTY_TAG_VALUE)
+      end
+
       def update_host_tags(host_name, options={})
         if Integer === host_name
           host_id = host_name
@@ -173,13 +231,16 @@ module Hotdog
         end
 
         if not options[:force]
-          @update_host_tags_q3 ||= @db.prepare("SELECT MIN(expires_at) FROM hosts_tags WHERE host_id = ?;")
+          # Update host tags less frequently.
+          # Don't need to run updates on every expiration.
+          @update_host_tags_q3 ||= @db.prepare("SELECT AVG(expires_at) FROM hosts_tags WHERE host_id = ?;")
           logger.debug("update_host_tags_q3(%s)" % [host_id.inspect])
           if expires_at = @update_host_tags_q3.execute(host_id).map { |row| row.first }.first
             if Time.new.to_i < expires_at
+              logger.debug("%s: next update will run after %s." % [host_name, Time.at(expires_at)])
               return
             else
-              logger.debug("%s: minimum expires_at was %s. start updating." % [host_name, Time.at(expires_at)])
+              logger.debug("%s: average expires_at was %s. start updating." % [host_name, Time.at(expires_at)])
             end
           else
             logger.debug("%s: expires_at not found. start updateing." % [host_name])
@@ -187,10 +248,12 @@ module Hotdog
         end
 
         code, result = @dog.host_tags(host_name)
+        logger.debug("dog.hosts_tags(%s) #==> [%s, %s]" % [host_name.inspect, code.inspect, result.inspect])
         if code.to_i / 100 != 2
           case code.to_i
           when 404 # host not found on datadog
             @update_host_tags_q7 ||= @db.prepare("DELETE FROM hosts_tags WHERE host_id IN ( SELECT id FROM hosts WHERE LOWER(name) = LOWER(?) );")
+            logger.debug("update_host_tags_q7(%s)" % [host_name.inspect])
             @update_host_tags_q7.execute(host_name)
           end
           raise("HTTP #{code}: #{result.inspect}")

data/lib/hotdog/version.rb

@@ -1,3 +1,3 @@
 module Hotdog
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hotdog
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Yamashita Yuu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-30 00:00:00.000000000 Z
+date: 2014-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler