hotdog 0.17.1 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bd0467d9928ac5366eec849cb7def564dd0d8ae3
-  data.tar.gz: 9588371163cbb077511b8b094360bb4ac4c7573d
+  metadata.gz: a32689688c7824af117417f98913a377e5f8e273
+  data.tar.gz: e814b609418012c7192cd8ada949a12392651288
 SHA512:
-  metadata.gz: 4f7f359862f6d3b7b79a89d03b04406e3a4efb07655084f9526346c1db88601b45252918a910e5c2081bafff8b8ef334d9555e37b12c2db73915dda1a2c1b748
-  data.tar.gz: cd802bb419145c53342d0409aa1623e376b7577aaa09be595f2474c13bae62a2b96470d258839b0e9c12f69c002dda5b7e98204c1b81b9a75342fc30a337663c
+  metadata.gz: 7b02bd4a8dc956cbcb74cf6854d6a540c197b2d4e96215b201aa4a06348d5ff6c9f3747edf4751ddda299f15b9400ebced4dd7ab11a498c7017d9a81d31a49ff
+  data.tar.gz: 062f80089aaa87daf0310578a728cceee13d9ad94d475fbb4be604553414831d42f0a8065a4a97896ed8a17d3f6d000f135948237cc8677d6a870d7ea6d7c5cf

data/lib/hotdog/application.rb

@@ -25,7 +25,7 @@ module Hotdog
         application: self,
         confdir: find_confdir(File.expand_path(".")),
         debug: false,
-        expiry: 1800,
+        expiry: 3600,
         fixed_string: false,
         force: false,
         format: "plain",

data/lib/hotdog/commands/down.rb

@@ -24,22 +24,29 @@ module Hotdog
       end
 
       def run(args=[], options={})
-        args.each do |arg|
+        scopes = args.map { |arg|
           if arg.index(":").nil?
-            scope = "host:#{arg}"
+            "host:#{arg}"
           else
-            scope = arg
+            arg
           end
+        }
+        scopes.each do |scope|
           with_retry(options) do
             schedule_downtime(scope, options)
           end
         end
-
-        # Remove persistent.db to schedule update on next invocation
-        if @db
-          close_db(@db)
+        hosts = scopes.select { |scope| scope.start_with?("host:") }.map { |scope|
+          scope.slice("host:".length, scope.length)
+        }
+        if 0 < hosts.length
+          if open_db
+            hosts.each_slice(SQLITE_LIMIT_COMPOUND_SELECT) do |hosts|
+              execute_db(@db, "DELETE FROM hosts_tags WHERE host_id IN ( SELECT id FROM hosts WHERE name IN (%s) )" % hosts.map { "?" }.join(", "), hosts)
+              execute_db(@db, "DELETE FROM hosts WHERE name IN (%s)" % hosts.map { "?" }.join(", "), hosts)
+            end
+          end
         end
-        FileUtils.rm_f(File.join(options[:confdir], PERSISTENT_DB))
       end
 
       private

data/lib/hotdog/commands/pssh.rb

@@ -34,15 +34,24 @@ module Hotdog
           infile.seek(0)
         end
         begin
-          stats = Parallel.map(hosts.each_with_index.to_a, in_threads: parallelism(hosts)) { |host, i|
-            cmdline = build_command_string(host, @remote_command, options)
-            identifier = options[:show_identifier] ? host : nil
-            success = exec_command(identifier, cmdline, index: i, output: true, infile: (infile ? infile.path : nil))
-            if !success && options[:stop_on_error]
-              raise StopException.new
-            end
-            success
+          hosts_cmdlines = hosts.map { |host|
+            [host, build_command_string(host, @remote_command, options)]
           }
+          if options[:dry_run]
+            stats = hosts_cmdlines.map { |host, cmdline|
+              STDOUT.puts(cmdline)
+              true
+            }
+          else
+            stats = Parallel.map(hosts_cmdlines.each_with_index.to_a, in_threads: parallelism(hosts)) { |(host, cmdline), i|
+              identifier = options[:show_identifier] ? host : nil
+              success = exec_command(identifier, cmdline, index: i, output: true, infile: (infile ? infile.path : nil))
+              if !success && options[:stop_on_error]
+                raise StopException.new
+              end
+              success
+            }
+          end
           if stats.all?
             exit(0)
           else

data/lib/hotdog/commands/scp.rb

@@ -13,32 +13,6 @@ module Hotdog
         cmdline = ["scp"] + build_command_options(options) + Shellwords.split(command).map { |token| token.gsub(/@(?=:)/, host) }
         Shellwords.join(cmdline)
       end
-
-      def build_command_options(options={})
-        arguments = []
-        if options[:forward_agent]
-          # nop
-        end
-        if options[:ssh_config]
-          cmdline << "-F" << File.expand_path(options[:ssh_config])
-        end
-        if options[:identity_file]
-          arguments << "-i" << options[:identity_file]
-        end
-        if options[:user]
-          arguments << "-o" << "User=#{options[:user]}"
-        end
-        if options[:options]
-          arguments += options[:options].flat_map { |option| ["-o", option] }
-        end
-        if options[:port]
-          arguments << "-P" << options[:port]
-        end
-        if options[:verbose]
-          arguments << "-v"
-        end
-        arguments
-      end
     end
   end
 end

data/lib/hotdog/commands/search.rb

@@ -36,11 +36,7 @@ module Hotdog
         if @remote_command
           logger.warn("ignore remote command: #{@remote_command}")
         end
-        expression = args.join(" ").strip
-        if expression.empty?
-          # return everything if given expression is empty
-          expression = "*"
-        end
+        expression = rewrite_expression(args.join(" ").strip)
 
         begin
           node = parse(expression)
@@ -114,6 +110,15 @@ module Hotdog
           raise("parser error: unknown expression: #{node.inspect}")
         end
       end
+
+      private
+      def rewrite_expression(expression)
+        if expression.empty?
+          # return everything if given expression is empty
+          expression = "*"
+        end
+        expression
+      end
     end
   end
 end

data/lib/hotdog/commands/sftp.rb

@@ -15,32 +15,6 @@ module Hotdog
         end
         Shellwords.join(cmdline)
       end
-
-      def build_command_options(options={})
-        arguments = []
-        if options[:forward_agent]
-          # nop
-        end
-        if options[:ssh_config]
-          cmdline << "-F" << File.expand_path(options[:ssh_config])
-        end
-        if options[:identity_file]
-          arguments << "-i" << options[:identity_file]
-        end
-        if options[:user]
-          arguments << "-o" << "User=#{options[:user]}"
-        end
-        if options[:options]
-          arguments += options[:options].flat_map { |option| ["-o", option] }
-        end
-        if options[:port]
-          arguments << "-P" << options[:port]
-        end
-        if options[:verbose]
-          arguments << "-v"
-        end
-        arguments
-      end
     end
   end
 end

data/lib/hotdog/commands/ssh.rb

@@ -9,32 +9,35 @@ module Hotdog
   module Commands
     class SshAlike < Search
       def define_options(optparse, options={})
-        default_option(options, :options, [])
-        default_option(options, :user, nil)
-        default_option(options, :port, nil)
-        default_option(options, :identity_file, nil)
-        default_option(options, :forward_agent, false)
+        default_option(options, :ssh_options, {})
         default_option(options, :color, :auto)
         default_option(options, :max_parallelism, Parallel.processor_count)
         default_option(options, :shuffle, false)
         default_option(options, :ssh_config, nil)
+        optparse.on("-C", "Enable compression.") do |v|
+          options[:ssh_options]["Compression"] = "yes"
+        end
         optparse.on("-F SSH_CONFIG", "Specifies an alternative per-user SSH configuration file.") do |configfile|
           options[:ssh_config] = configfile
         end
-        optparse.on("-o SSH_OPTION", "Passes this string to ssh command through shell. This option may be given multiple times") do |option|
-          options[:options] += [option]
+        optparse.on("--dry-run", "Dry run.") do |v|
+          options[:dry_run] = v
+        end
+        optparse.on("-o SSH_OPTION", "Passes this string to ssh command through shell. This option may be given multiple times") do |ssh_option|
+          ssh_option_key, ssh_option_value = ssh_option.split("=", 2)
+          options[:ssh_options][ssh_option_key] = ssh_option_value
         end
         optparse.on("-i SSH_IDENTITY_FILE", "SSH identity file path") do |path|
-          options[:identity_file] = path
+          options[:ssh_options]["IdentityFile"] = path
         end
         optparse.on("-A", "Enable agent forwarding", TrueClass) do |b|
-          options[:forward_agent] = b
+          options[:ssh_options]["ForwardAgent"] = "yes"
         end
         optparse.on("-p PORT", "Port of the remote host", Integer) do |port|
-          options[:port] = port
+          options[:ssh_options]["Port"] = port
         end
         optparse.on("-u SSH_USER", "SSH login user name") do |user|
-          options[:user] = user
+          options[:ssh_options]["User"] = user
         end
         optparse.on("-v", "--verbose", "Enable verbose ode") do |v|
           options[:verbose] = v
@@ -54,11 +57,7 @@ module Hotdog
       end
 
       def run(args=[], options={})
-        expression = args.join(" ").strip
-        if expression.empty?
-          # return everything if given expression is empty
-          expression = "*"
-        end
+        expression = rewrite_expression(args.join(" ").strip)
 
         begin
           node = parse(expression)
@@ -69,15 +68,20 @@ module Hotdog
 
         result0 = evaluate(node, self)
         tuples, fields = get_hosts_with_search_tags(result0, node)
-        if options[:shuffle]
-          tuples = tuples.shuffle()
-        end
         tuples = filter_hosts(tuples)
         validate_hosts!(tuples, fields)
         run_main(tuples.map {|tuple| tuple.first }, options)
       end
 
       private
+      def rewrite_expression(expression)
+        expression = super(expression)
+        if options[:shuffle]
+          expression = "SHUFFLE((#{expression}))"
+        end
+        expression
+      end
+
       def parallelism(hosts)
         options[:max_parallelism] || hosts.size
       end
@@ -116,24 +120,10 @@ module Hotdog
 
       def build_command_options(options={})
         cmdline = []
-        if options[:forward_agent]
-          cmdline << "-A"
-        end
         if options[:ssh_config]
           cmdline << "-F" << File.expand_path(options[:ssh_config])
         end
-        if options[:identity_file]
-          cmdline << "-i" << options[:identity_file]
-        end
-        if options[:user]
-          cmdline << "-l" << options[:user]
-        end
-        if options[:options]
-          cmdline += options[:options].flat_map { |option| ["-o", option] }
-        end
-        if options[:port]
-          cmdline << "-p" << options[:port].to_s
-        end
+        cmdline += options[:ssh_options].flat_map { |k, v| ["-o", "#{k}=#{v}"] }
         if options[:verbose]
           cmdline << "-v"
         end
@@ -210,6 +200,14 @@ module Hotdog
       end
 
       private
+      def rewrite_expression(expression)
+        expression = super(expression)
+        if options[:index]
+          expression = "SLICE((#{expression}), #{options[:index]}, 1)"
+        end
+        expression
+      end
+
       def filter_hosts(tuples)
         tuples = super
         if options[:index] and options[:index] < tuples.length
@@ -232,7 +230,12 @@ module Hotdog
       def run_main(hosts, options={})
         cmdline = build_command_string(hosts.first, @remote_command, options)
         logger.debug("execute: #{cmdline}")
-        exec(cmdline)
+        if options[:dry_run]
+          STDOUT.puts(cmdline)
+          exit(0)
+        else
+          exec(cmdline)
+        end
         exit(127)
       end
     end

data/lib/hotdog/commands/tag.rb

@@ -24,24 +24,25 @@ module Hotdog
       end
 
       def run(args=[], options={})
-        args.each do |host_name|
-          host_name = host_name.sub(/\Ahost:/, "")
-
+        hosts = args.map { |arg|
+          arg.sub(/\Ahost:/, "")
+        }
+        hosts.each do |host|
           if options[:tags].empty?
             # nop
           else
             # add all as user tags
             with_retry(options) do
-              add_tags(host_name, options[:tags], source=options[:tag_source])
+              add_tags(host, options[:tags], source=options[:tag_source])
             end
           end
         end
-
-        # Remove persistent.db to schedule update on next invocation
-        if @db
-          close_db(@db)
+        if open_db
+          create_tags(@db, options[:tags])
+          options[:tags].each do |tag|
+            associae_tag_hosts(@db, tag, hosts)
+          end
         end
-        FileUtils.rm_f(File.join(options[:confdir], PERSISTENT_DB))
       end
 
       private

data/lib/hotdog/commands/untag.rb

@@ -24,33 +24,40 @@ module Hotdog
       end
 
       def run(args=[], options={})
-        args.each do |host_name|
-          host_name = host_name.sub(/\Ahost:/, "")
+        hosts = args.map { |arg|
+          arg.sub(/\Ahost:/, "")
+        }
 
+        hosts.each do |host|
           if options[:tags].empty?
             # delete all user tags
             with_retry do
-              detach_tags(host_name, source=options[:tag_source])
+              detach_tags(host, source=options[:tag_source])
             end
           else
-            host_tags = with_retry { host_tags(host_name, source=options[:tag_source]) }
+            host_tags = with_retry { host_tags(host, source=options[:tag_source]) }
             old_tags = host_tags["tags"]
             new_tags = old_tags - options[:tags]
             if old_tags == new_tags
               # nop
             else
               with_retry do
-                update_tags(host_name, new_tags, source=options[:tag_source])
+                update_tags(host, new_tags, source=options[:tag_source])
               end
             end
           end
         end
 
-        # Remove persistent.db to schedule update on next invocation
-        if @db
-          close_db(@db)
+        if options[:tags].empty?
+          # refresh all persistent.db since there is no way to identify user tags
+          remove_db
+        else
+          if open_db
+            options[:tags].each do |tag|
+              disassociate_tag_hosts(tag, hosts)
+            end
+          end
         end
-        FileUtils.rm_f(File.join(options[:confdir], PERSISTENT_DB))
       end
 
       private

data/lib/hotdog/commands/up.rb

@@ -37,12 +37,13 @@ module Hotdog
             cancel_downtime(downtime["id"], options)
           end
         end
-
-        # Remove persistent.db to schedule update on next invocation
-        if @db
-          close_db(@db)
+        hosts = scopes.select { |scope| scope.start_with?("host:") }.map { |scope|
+          scope.slice("host:".length, scope.length)
+        }
+        if 0 < hosts.length
+          # refresh all persistent.db to retrieve information about up'd host
+          remove_db
         end
-        FileUtils.rm_f(File.join(options[:confdir], PERSISTENT_DB))
       end
 
       private

data/lib/hotdog/commands.rb

@@ -92,25 +92,16 @@ module Hotdog
             get_hosts_fields(host_ids, fields)
           else
             if @options[:listing]
-              q1 = "SELECT DISTINCT tags.name FROM hosts_tags " \
-                     "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
-                       "WHERE hosts_tags.host_id IN (%s);"
               if @options[:primary_tag]
                 fields = [
                   @options[:primary_tag],
                   "host",
-                ] + host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
-                  execute(q1 % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.first }.reject { |tag_name|
-                    tag_name == @options[:primary_tag]
-                  }
-                }
+                ] + get_fields(host_ids).reject { |tag_name| tag_name == @options[:primary_tag] }
                 get_hosts_fields(host_ids, fields)
               else
                 fields = [
                   "host",
-                ] + host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
-                  execute(q1 % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.first }
-                }
+                ] + get_fields(host_ids)
                 get_hosts_fields(host_ids, fields)
               end
             else
@@ -124,48 +115,85 @@ module Hotdog
         end
       end
 
-      def get_hosts_fields(host_ids, fields)
-        if fields.empty?
+      def get_fields(host_ids)
+        host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
+          q = "SELECT DISTINCT tags.name FROM hosts_tags " \
+                "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
+                  "WHERE hosts_tags.host_id IN (%s);" % host_ids.map { "?" }.join(", ")
+          execute(q, host_ids).map { |row| row.first }
+        }.uniq
+      end
+
+      def get_hosts_fields(host_ids, fields, options={})
+        case fields.length
+        when 0
           [[], fields]
+        when 1
+          get_hosts_field(host_ids, fields.first, options)
         else
-          fields_without_host = fields.reject { |tag_name| tag_name == "host" }
-          if fields == fields_without_host
-            host_names = {}
+          if fields.find { |field| /\Ahost\z/i =~ field }
+            host_names = Hash[execute("SELECT id, name FROM hosts WHERE id IN (%s);" % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.to_a }]
           else
-            host_names = Hash[
-              host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
-                execute("SELECT id, name FROM hosts WHERE id IN (%s)" % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.to_a }
-              }
-            ]
+            host_names = {}
           end
-          q1 = "SELECT LOWER(tags.name), GROUP_CONCAT(tags.value, ',') FROM hosts_tags " \
-                 "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
-                   "WHERE hosts_tags.host_id = ? AND tags.name IN (%s) " \
-                     "GROUP BY tags.name;"
-          result = host_ids.map { |host_id|
-            tag_values = Hash[
-              fields_without_host.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 1).flat_map { |fields_without_host|
-                execute(q1 % fields_without_host.map { "?" }.join(", "), [host_id] + fields_without_host).map { |row| row.to_a }
-              }
-            ]
-            fields.map { |tag_name|
-              if tag_name == "host"
-                host_names.fetch(host_id, nil)
-              else
-                tag_value = tag_values.fetch(tag_name.downcase, nil)
-                if tag_value
-                  if tag_value.empty?
-                    tag_name # use `tag_name` as `tag_value` for the tags without any values
-                  else
-                    tag_value
-                  end
-                else
-                  nil
-                end
-              end
+
+          [host_ids.map { |host_id| get_host_fields(host_id, fields, options.merge(host_names: host_names)) }.map { |result, fields| result }, fields]
+        end
+      end
+
+      def get_host_fields(host_id, fields, options={})
+        field_values = {"host" => options.fetch(:host_names, {}).fetch(host_id, nil)}
+
+        fields.reject { |field| /\Ahost\z/i =~ field }.uniq.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 1).each do |fields|
+          q = "SELECT LOWER(tags.name), GROUP_CONCAT(tags.value, ',') FROM hosts_tags " \
+                "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
+                  "WHERE hosts_tags.host_id = ? AND tags.name IN (%s) " \
+                    "GROUP BY tags.name;" % fields.map { "?" }.join(", ")
+          execute(q, [host_id] + fields).each do |row|
+            field_values[row[0]] = row[1]
+          end
+        end
+
+        result = fields.map { |tag_name|
+          tag_value = field_values.fetch(tag_name.downcase, nil)
+          display_tag(tag_name, tag_value)
+        }
+        [result, fields]
+      end
+
+      def get_hosts_field(host_ids, field, options={})
+        if /\Ahost\z/i =~ field
+          result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT).flat_map { |host_ids|
+            execute("SELECT name FROM hosts WHERE id IN (%s)" % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.to_a }
+          }
+        else
+          result = host_ids.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 1).flat_map { |host_ids|
+            q = "SELECT LOWER(tags.name), GROUP_CONCAT(tags.value, ',') FROM hosts_tags " \
+                  "INNER JOIN tags ON hosts_tags.tag_id = tags.id " \
+                    "WHERE hosts_tags.host_id IN (%s) AND tags.name = ? " \
+                      "GROUP BY hosts_tags.host_id, tags.name;" % host_ids.map { "?" }.join(", ")
+            r = execute(q, host_ids + [field]).map { |tag_name, tag_value|
+              [display_tag(tag_name, tag_value)]
             }
+            if r.empty?
+              host_ids.map { [nil] }
+            else
+              r
+            end
           }
-          [result, fields]
+        end
+        [result, [field]]
+      end
+
+      def display_tag(tag_name, tag_value)
+        if tag_value
+          if tag_value.empty?
+            tag_name # use `tag_name` as `tag_value` for the tags without any values
+          else
+            tag_value
+          end
+        else
+          nil
         end
       end
 
@@ -178,23 +206,19 @@ module Hotdog
         db.close()
       end
 
-      def update_db(options={})
+      def open_db(options={})
         options = @options.merge(options)
-        if @db.nil?
+        if @db
+          @db
+        else
           FileUtils.mkdir_p(options[:confdir])
           persistent = File.join(options[:confdir], PERSISTENT_DB)
 
           if (not options[:force] and File.exist?(persistent) and Time.new < File.mtime(persistent) + options[:expiry]) or options[:offline]
             begin
               persistent_db = SQLite3::Database.new(persistent)
-              persistent_db.execute(<<-EOS)
-                SELECT hosts_tags.host_id FROM hosts_tags
-                  INNER JOIN hosts ON hosts_tags.host_id = hosts.id
-                  INNER JOIN tags ON hosts_tags.tag_id = tags.id
-                    LIMIT 1;
-              EOS
+              persistent_db.execute("SELECT hosts_tags.host_id FROM hosts_tags INNER JOIN hosts ON hosts_tags.host_id = hosts.id INNER JOIN tags ON hosts_tags.tag_id = tags.id LIMIT 1;")
               @db = persistent_db
-              return
             rescue SQLite3::SQLException
               if options[:offline]
                 raise(RuntimeError.new("no database available on offline mode"))
@@ -203,81 +227,56 @@ module Hotdog
               end
             end
           end
+          @db
+        end
+      end
 
+      def update_db(options={})
+        options = @options.merge(options)
+        if open_db(options)
+          @db
+        else
           memory_db = SQLite3::Database.new(":memory:")
-          execute_db(memory_db, <<-EOS)
-            CREATE TABLE IF NOT EXISTS hosts (
-              id INTEGER PRIMARY KEY AUTOINCREMENT,
-              name VARCHAR(255) NOT NULL COLLATE NOCASE
-            );
-          EOS
-          execute_db(memory_db, "CREATE UNIQUE INDEX IF NOT EXISTS hosts_name ON hosts ( name );")
-          execute_db(memory_db, <<-EOS)
-            CREATE TABLE IF NOT EXISTS tags (
-              id INTEGER PRIMARY KEY AUTOINCREMENT,
-              name VARCHAR(200) NOT NULL COLLATE NOCASE,
-              value VARCHAR(200) NOT NULL COLLATE NOCASE
-            );
-          EOS
-          execute_db(memory_db, "CREATE UNIQUE INDEX IF NOT EXISTS tags_name_value ON tags ( name, value );")
-          execute_db(memory_db, <<-EOS)
-            CREATE TABLE IF NOT EXISTS hosts_tags (
-              host_id INTEGER NOT NULL,
-              tag_id INTEGER NOT NULL
-            );
-          EOS
-          execute_db(memory_db, "CREATE UNIQUE INDEX IF NOT EXISTS hosts_tags_host_id_tag_id ON hosts_tags ( host_id, tag_id );")
+          execute_db(memory_db, "CREATE TABLE IF NOT EXISTS hosts (id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(255) NOT NULL COLLATE NOCASE);")
+          execute_db(memory_db, "CREATE UNIQUE INDEX IF NOT EXISTS hosts_name ON hosts (name);")
+          execute_db(memory_db, "CREATE TABLE IF NOT EXISTS tags (id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR(200) NOT NULL COLLATE NOCASE, value VARCHAR(200) NOT NULL COLLATE NOCASE);")
+          execute_db(memory_db, "CREATE UNIQUE INDEX IF NOT EXISTS tags_name_value ON tags (name, value);")
+          execute_db(memory_db, "CREATE TABLE IF NOT EXISTS hosts_tags (host_id INTEGER NOT NULL, tag_id INTEGER NOT NULL);")
+          execute_db(memory_db, "CREATE UNIQUE INDEX IF NOT EXISTS hosts_tags_host_id_tag_id ON hosts_tags (host_id, tag_id);")
 
           all_tags = get_all_tags()
 
           memory_db.transaction do
             known_tags = all_tags.keys.map { |tag| split_tag(tag) }.uniq
-            known_tags.each_slice(SQLITE_LIMIT_COMPOUND_SELECT / 2) do |known_tags|
-              q = "INSERT OR IGNORE INTO tags (name, value) VALUES %s" % known_tags.map { "(?, ?)" }.join(", ")
-              execute_db(memory_db, q, known_tags)
-            end
+            create_tags(memory_db, known_tags)
 
             known_hosts = all_tags.values.reduce(:+).uniq
-            known_hosts.each_slice(SQLITE_LIMIT_COMPOUND_SELECT) do |known_hosts|
-              q = "INSERT OR IGNORE INTO hosts (name) VALUES %s" % known_hosts.map { "(?)" }.join(", ")
-              execute_db(memory_db, q, known_hosts)
-            end
+            create_hosts(memory_db, known_hosts)
 
             all_tags.each do |tag, hosts|
-              hosts.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 2) do |hosts|
-                q = "INSERT OR REPLACE INTO hosts_tags (host_id, tag_id) " \
-                      "SELECT host.id, tag.id FROM " \
-                        "( SELECT id FROM hosts WHERE name IN (%s) ) AS host, " \
-                        "( SELECT id FROM tags WHERE name = ? AND value = ? LIMIT 1 ) AS tag;" % hosts.map { "?" }.join(", ")
-                begin
-                  execute_db(memory_db, q, (hosts + split_tag(tag)))
-                rescue SQLite3::RangeException => error
-                  # FIXME: bulk insert occationally fails even if there are no errors in bind parameters
-                  #        `bind_param': bind or column index out of range (SQLite3::RangeException)
-                  logger.warn("bulk insert failed due to #{error.message}. fallback to normal insert.")
-                  hosts.each do |host|
-                    q = "INSERT OR REPLACE INTO hosts_tags (host_id, tag_id) " \
-                          "SELECT host.id, tag.id FROM " \
-                            "( SELECT id FROM hosts WHERE name = ? ) AS host, " \
-                            "( SELECT id FROM tags WHERE name = ? AND value = ? LIMIT 1 ) AS tag;"
-                    execute_db(memory_db, q, [host] + split_tag(tag))
-                  end
-                end
-              end
+              associate_tag_hosts(memory_db, tag, hosts)
             end
           end
 
           # backup in-memory db to file
+          FileUtils.mkdir_p(options[:confdir])
+          persistent = File.join(options[:confdir], PERSISTENT_DB)
           FileUtils.rm_f(persistent)
           persistent_db = SQLite3::Database.new(persistent)
           copy_db(memory_db, persistent_db)
-          close_db(persistent_db)
-          @db = memory_db
-        else
-          @db
+          close_db(memory_db)
+          @db = persistent_db
         end
       end
 
+      def remove_db(options={})
+        options = @options.merge(options)
+        if @db
+          close_db(@db)
+        end
+        FileUtils.rm_f(File.join(options[:confdir], PERSISTENT_DB))
+      end
+
       def execute_db(db, q, args=[])
         begin
           logger.debug("execute: #{q} -- #{args.inspect}")
@@ -322,6 +321,52 @@ module Hotdog
         Hash[responses.fetch(:all_tags, {}).fetch("tags", []).map { |tag, hosts| [tag, hosts.reject { |host| downtimes.include?(host) }] }]
       end
 
+      def create_hosts(db, hosts)
+        hosts.each_slice(SQLITE_LIMIT_COMPOUND_SELECT) do |hosts|
+          q = "INSERT OR IGNORE INTO hosts (name) VALUES %s" % hosts.map { "(?)" }.join(", ")
+          execute_db(db, q, hosts)
+        end
+      end
+
+      def create_tags(db, tags)
+        tags.each_slice(SQLITE_LIMIT_COMPOUND_SELECT / 2) do |tags|
+          q = "INSERT OR IGNORE INTO tags (name, value) VALUES %s" % tags.map { "(?, ?)" }.join(", ")
+          execute_db(db, q, tags)
+        end
+      end
+
+      def associate_tag_hosts(db, tag, hosts)
+        hosts.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 2) do |hosts|
+          q = "INSERT OR REPLACE INTO hosts_tags (host_id, tag_id) " \
+                "SELECT host.id, tag.id FROM " \
+                  "( SELECT id FROM hosts WHERE name IN (%s) ) AS host, " \
+                  "( SELECT id FROM tags WHERE name = ? AND value = ? LIMIT 1 ) AS tag;" % hosts.map { "?" }.join(", ")
+          begin
+            execute_db(db, q, (hosts + split_tag(tag)))
+          rescue SQLite3::RangeException => error
+            # FIXME: bulk insert occationally fails even if there are no errors in bind parameters
+            #        `bind_param': bind or column index out of range (SQLite3::RangeException)
+            logger.warn("bulk insert failed due to #{error.message}. fallback to normal insert.")
+            hosts.each do |host|
+              q = "INSERT OR REPLACE INTO hosts_tags (host_id, tag_id) " \
+                    "SELECT host.id, tag.id FROM " \
+                      "( SELECT id FROM hosts WHERE name = ? ) AS host, " \
+                      "( SELECT id FROM tags WHERE name = ? AND value = ? LIMIT 1 ) AS tag;"
+              execute_db(db, q, [host] + split_tag(tag))
+            end
+          end
+        end
+      end
+
+      def disassociate_tag_hosts(db, tag, hosts)
+        hosts.each_slice(SQLITE_LIMIT_COMPOUND_SELECT - 2) do |hosts|
+          q = "DELETE FROM hosts_tags " \
+                "WHERE tag_id IN ( SELECT id FROM tags WHERE name = ? AND value = ? LIMIT 1 );" \
+                  "AND host_id IN ( SELECT id FROM hosts WHERE name IN (%s) ) " % hosts.map { "?" }.join(", ")
+          execute_db(db, q, split_tag(tag) + hosts)
+        end
+      end
+
       def dog()
         @dog ||= Dogapi::Client.new(application.api_key, application.application_key)
       end

data/lib/hotdog/expression/semantics.rb

@@ -425,8 +425,13 @@ module Hotdog
           @function = :ORDER_BY
         when "REVERSE", "reverse"
           @function = :REVERSE
+        when "SAMPLE", "sample"
+          @function = :HEAD
+          args[0] = FuncallNode.new("SHUFFLE", [args[0]])
         when "SHUFFLE", "shuffle"
           @function = :SHUFFLE
+        when "SLICE", "slice"
+          @function = :SLICE
         when "SORT", "sort"
           @function = :ORDER_BY
         when "TAIL", "tail"
@@ -474,6 +479,8 @@ module Hotdog
           @args[0] = @args[0].optimize(options)
         when :SHUFFLE
           @args[0] = @args[0].optimize(options)
+        when :SLICE
+          @args[0] = @args[0].optimize(options)
         when :TAIL
           @args[0] = @args[0].optimize(options)
         end
@@ -510,6 +517,8 @@ module Hotdog
           args[0].evaluate(environment, options).reverse()
         when :SHUFFLE
           args[0].evaluate(environment, options).shuffle()
+        when :SLICE
+          args[0].evaluate(environment, options).slice(args[1], args[2] || 1)
         when :TAIL
           args[0].evaluate(environment, options).last(args[1] || 1)
         else

data/lib/hotdog/expression/syntax.rb

@@ -87,7 +87,7 @@ module Hotdog
         )
       }
       rule(:integer) {
-        ( match('[1-9]').repeat(1) >> match('[0-9]').repeat(0) \
+        ( match('[0-9]').repeat(1) \
         )
       }
       rule(:string) {

data/lib/hotdog/version.rb

@@ -1,3 +1,3 @@
 module Hotdog
-  VERSION = "0.17.1"
+  VERSION = "0.18.0"
 end

data/spec/core/commands_spec.rb

@@ -1,6 +1,7 @@
 require "spec_helper"
 require "hotdog/application"
 require "hotdog/commands"
+require "hotdog/commands/search"
 
 describe "commands" do
   let(:cmd) {
@@ -99,7 +100,7 @@ describe "commands" do
   end
 
   it "get host fields with host" do
-    allow(cmd).to receive(:execute).with("SELECT id, name FROM hosts WHERE id IN (?, ?, ?)", [1, 2, 3]) {
+    allow(cmd).to receive(:execute).with("SELECT id, name FROM hosts WHERE id IN (?, ?, ?);", [1, 2, 3]) {
       [[1, "host1"], [2, "host2"], [3, "host3"]]
     }
     q1 = [

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hotdog
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 0.18.0
 platform: ruby
 authors:
 - Yamashita Yuu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-29 00:00:00.000000000 Z
+date: 2016-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler