hotdog 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 017715f50e986cf3c0f61cb4834859c296ddf6f7
-  data.tar.gz: 137f2e8504b9c9908635b8dfe7db6d2b1541be9b
+  metadata.gz: 551436c6cb264d187db243f60da58a14f38cb820
+  data.tar.gz: f68bd1dd186fa326af8541b9ea3ae4ae95075d44
 SHA512:
-  metadata.gz: 3b406633a8f174b328dca17c976b4585d6e187f8cefc2dfe7c19ca094f2469faf292a1335d8aaea75f8e440c7d24f1878782710ac9b4c8456cea65466ced87cc
-  data.tar.gz: a1f2843dd806045110b64c915c8fda049f1a228654e8fc1282f5aa057280e129ecd5da06ca88a80db37b0f35c3c3054ec6e57338fb20d0e9ca176169fb17811c
+  metadata.gz: f85d09c3f08d2cce6fec0d9d73654e86b2ac6fa8abaa6b2682047f8b4daeeedb6985dc7a0f4613bee2076caf43ecf8e0df129f45023d986b14b306ee971a2a38
+  data.tar.gz: 2e983f2f1508ea69872aa58758ee2a7cfca55e3dcbcba2b612f26e8b543fb8b87a0b849b0d5ef198fb4919c1622c5f2f90bdeb38d73f0ea5ead1e23ad987c11c

data/lib/hotdog/commands/search.rb

@@ -242,16 +242,24 @@ module Hotdog
           case @op
           when :AND
             left_values = @left.evaluate(environment)
+            environment.logger.debug("lhs: #{left_values.length} value(s)")
             if left_values.empty?
               []
             else
               right_values = @right.evaluate(environment)
-              (left_values & right_values)
+              environment.logger.debug("rhs: #{right_values.length} value(s)")
+              (left_values & right_values).tap do |values|
+                environment.logger.debug("lhs AND rhs: #{values.length} value(s)")
+              end
             end
           when :OR
             left_values = @left.evaluate(environment)
+            environment.logger.debug("lhs: #{left_values.length} value(s)")
             right_values = @right.evaluate(environment)
-            (left_values | right_values).uniq
+            environment.logger.debug("rhs: #{right_values.length} value(s)")
+            (left_values | right_values).uniq.tap do |values|
+              environment.logger.debug("lhs OR rhs: #{values.length} value(s)")
+            end
           else
             []
           end
@@ -288,11 +296,23 @@ module Hotdog
         def evaluate(environment, options={})
           case @op
           when :NOT
-            values = @expression.evaluate(environment)
+            values = @expression.evaluate(environment).sort
+            environment.logger.debug("expr: #{values.length} value(s)")
             if values.empty?
-              environment.execute("SELECT DISTINCT host_id FROM hosts_tags").map { |row| row.first }
+              environment.execute("SELECT id FROM hosts").map { |row| row.first }.tap do |values|
+                environment.logger.debug("NOT expr: #{values.length} value(s)")
+              end
             else
-              environment.execute("SELECT DISTINCT host_id FROM hosts_tags WHERE host_id NOT IN (%s)" % values.map { "?" }.join(", "), values).map { |row| row.first }
+              # workaround for "too many terms in compound SELECT"
+              min, max = environment.execute("SELECT MIN(id), MAX(id) FROM hosts ORDER BY id LIMIT 1").first.to_a
+              slice = 200
+              (min / slice).upto(max / slice).map { |i|
+                range = (slice*i)...(slice*(i+1))
+                selected = values.select { |n| range === n }
+                environment.execute("SELECT id FROM hosts WHERE ? <= id AND id < ? AND id NOT IN (%s)" % selected.map { "?" }.join(", "), [range.first, range.last] + selected).map { |row| row.first }
+              }.reduce(:+).tap do |values|
+                environment.logger.debug("NOT expr: #{values.length} value(s)")
+              end
             end
           else
             []

data/lib/hotdog/commands.rb

@@ -11,6 +11,7 @@ module Hotdog
       PERSISTENT_DB = "persistent.db"
       MASK_DATABASE = 0xffff0000
       MASK_QUERY = 0x0000ffff
+      MAX_TERMS = 200
 
       def initialize(application)
         @application = application
@@ -82,7 +83,11 @@ module Hotdog
             if fields == fields_without_host
               host_names = {}
             else
-              host_names = Hash[execute("SELECT id, name FROM hosts WHERE id IN (%s)" % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.to_a }]
+              host_names = Hash[
+                host_ids.each_slice(MAX_TERMS).map { |host_ids|
+                  execute("SELECT id, name FROM hosts WHERE id IN (%s)" % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.to_a }
+                }.reduce(:+)
+              ]
             end
             q1 = []
             q1 << "SELECT tags.name, GROUP_CONCAT(tags.value, ',') FROM hosts_tags"
@@ -91,7 +96,11 @@ module Hotdog
             q1 <<     "WHERE hosts_tags.host_id = ? AND tags.name IN (%s)"
             q1 <<       "GROUP BY tags.name;"
             result = host_ids.map { |host_id|
-              tag_values = Hash[execute(q1.join(" ") % fields_without_host.map { "?" }.join(", "), [host_id] + fields_without_host).map { |row| row.to_a }]
+              tag_values = Hash[
+                fields_without_host.each_slice(MAX_TERMS).map { |fields_without_host|
+                  execute(q1.join(" ") % fields_without_host.map { "?" }.join(", "), [host_id] + fields_without_host).map { |row| row.to_a }
+                }.reduce(:+)
+              ]
               fields.map { |tag_name|
                 if tag_name == "host"
                   host_names.fetch(host_id, "")
@@ -111,15 +120,23 @@ module Hotdog
                 fields = [
                   @options[:primary_tag],
                   "host",
-                ] + execute(q1.join(" ") % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.first }.reject { |tag_name|
-                  tag_name == @options[:primary_tag]
-                }
+                ] + host_ids.each_slice(MAX_TERMS).map { |host_ids|
+                  execute(q1.join(" ") % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.first }.reject { |tag_name|
+                    tag_name == @options[:primary_tag]
+                  }
+                }.reduce(:+)
               else
                 fields = [
                   "host",
-                ] + execute(q1.join(" ") % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.first }
+                ] + host_ids.each_slice(MAX_TERMS).map { |host_ids|
+                  execute(q1.join(" ") % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.first }
+                }.reduce(:+)
               end
-              host_names = Hash[execute("SELECT id, name FROM hosts WHERE id IN (%s)" % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.to_a }]
+              host_names = Hash[
+                host_ids.each_slice(MAX_TERMS).map { |host_ids|
+                  execute("SELECT id, name FROM hosts WHERE id IN (%s)" % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.to_a }
+                }.reduce(:+)
+              ]
               q2 = []
               q2 << "SELECT tags.name, GROUP_CONCAT(tags.value, ',') FROM hosts_tags"
               q2 <<   "INNER JOIN tags ON hosts_tags.tag_id = tags.id"
@@ -127,7 +144,11 @@ module Hotdog
               q2 <<       "GROUP BY tags.name;"
               fields_without_host = fields.reject { |tag_name| tag_name == "host" }
               result = host_ids.map { |host_id|
-                tag_values = Hash[execute(q2.join(" ") % fields_without_host.map { "?" }.join(", "), [host_id] + fields_without_host).map { |row| row.to_a }]
+                tag_values = Hash[
+                  fields_without_host.each_slice(MAX_TERMS).map { |fields_without_host|
+                    execute(q2.join(" ") % fields_without_host.map { "?" }.join(", "), [host_id] + fields_without_host).map { |row| row.to_a }
+                  }.reduce(:+)
+                ]
                 fields.map { |tag_name|
                   if tag_name == "host"
                     host_names.fetch(host_id, "")
@@ -145,11 +166,15 @@ module Hotdog
                 q1 <<   "INNER JOIN hosts ON hosts_tags.host_id = hosts.id"
                 q1 <<   "INNER JOIN tags ON hosts_tags.tag_id = tags.id"
                 q1 <<     "WHERE hosts_tags.host_id IN (%s) AND tags.name = ?;"
-                result = execute(q1.join(" ") % host_ids.map { "?" }.join(", "), host_ids + [@options[:primary_tag]]).map { |row| row.to_a }
+                result = host_ids.each_slice(MAX_TERMS).map { |host_ids|
+                  execute(q1.join(" ") % host_ids.map { "?" }.join(", "), host_ids + [@options[:primary_tag]]).map { |row| row.to_a }
+                }.reduce(:+)
                 [result, fields]
               else
                 fields = ["host"]
-                result = execute("SELECT name FROM hosts WHERE id IN (%s)" % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.to_a }
+                result = host_ids.each_slice(MAX_TERMS).map { |host_ids|
+                  execute("SELECT name FROM hosts WHERE id IN (%s)" % host_ids.map { "?" }.join(", "), host_ids).map { |row| row.to_a }
+                }.reduce(:+)
                 [result, fields]
               end
             end
@@ -195,24 +220,26 @@ module Hotdog
 
           all_tags = get_all_tags()
 
-          known_tags = all_tags.keys.map { |tag| split_tag(tag) }.uniq
-          unless known_tags.empty?
-            prepare(memory_db, "INSERT OR IGNORE INTO tags (name, value) VALUES %s" % known_tags.map { "(?, ?)" }.join(", ")).execute(known_tags)
-          end
+          memory_db.transaction do
+            known_tags = all_tags.keys.map { |tag| split_tag(tag) }.uniq
+            known_tags.each_slice(MAX_TERMS) do |known_tags|
+              prepare(memory_db, "INSERT OR IGNORE INTO tags (name, value) VALUES %s" % known_tags.map { "(?, ?)" }.join(", ")).execute(known_tags)
+            end
 
-          known_hosts = all_tags.values.reduce(:+).uniq
-          unless known_hosts.empty?
-            prepare(memory_db, "INSERT OR IGNORE INTO hosts (name) VALUES %s" % known_hosts.map { "(?)" }.join(", ")).execute(known_hosts)
-          end
+            known_hosts = all_tags.values.reduce(:+).uniq
+            known_hosts.each_slice(MAX_TERMS) do |known_hosts|
+              prepare(memory_db, "INSERT OR IGNORE INTO hosts (name) VALUES %s" % known_hosts.map { "(?)" }.join(", ")).execute(known_hosts)
+            end
 
-          all_tags.each do |tag, hosts|
-            q = []
-            q << "INSERT OR REPLACE INTO hosts_tags (host_id, tag_id)"
-            q <<   "SELECT host.id, tag.id FROM"
-            q <<     "( SELECT id FROM hosts WHERE name IN (%s) ) AS host,"
-            q <<     "( SELECT id FROM tags WHERE name = ? AND value = ? LIMIT 1 ) AS tag;"
-            unless hosts.empty?
-              prepare(memory_db, q.join(" ") % hosts.map { "?" }.join(", ")).execute(hosts + split_tag(tag))
+            all_tags.each do |tag, hosts|
+              q = []
+              q << "INSERT OR REPLACE INTO hosts_tags (host_id, tag_id)"
+              q <<   "SELECT host.id, tag.id FROM"
+              q <<     "( SELECT id FROM hosts WHERE name IN (%s) ) AS host,"
+              q <<     "( SELECT id FROM tags WHERE name = ? AND value = ? LIMIT 1 ) AS tag;"
+              hosts.each_slice(MAX_TERMS) do |hosts|
+                prepare(memory_db, q.join(" ") % hosts.map { "?" }.join(", ")).execute(hosts + split_tag(tag))
+              end
             end
           end
 
@@ -265,13 +292,19 @@ module Hotdog
           create_table_hosts_tags(dst)
 
           hosts = prepare(src, "SELECT id, name FROM hosts").execute().to_a
-          prepare(dst, "INSERT INTO hosts (id, name) VALUES %s" % hosts.map { "(?, ?)" }.join(", ")).execute(hosts) unless hosts.empty?
+          hosts.each_slice(MAX_TERMS) do |hosts|
+            prepare(dst, "INSERT INTO hosts (id, name) VALUES %s" % hosts.map { "(?, ?)" }.join(", ")).execute(hosts)
+          end
 
           tags = prepare(src, "SELECT id, name, value FROM tags").execute().to_a
-          prepare(dst, "INSERT INTO tags (id, name, value) VALUES %s" % tags.map { "(?, ?, ?)" }.join(", ")).execute(tags) unless tags.empty?
+          tags.each_slice(MAX_TERMS) do |tags|
+            prepare(dst, "INSERT INTO tags (id, name, value) VALUES %s" % tags.map { "(?, ?, ?)" }.join(", ")).execute(tags)
+          end
 
           hosts_tags = prepare(src, "SELECT host_id, tag_id FROM hosts_tags").to_a
-          prepare(dst, "INSERT INTO hosts_tags (host_id, tag_id) VALUES %s" % hosts_tags.map { "(?, ?)" }.join(", ")).execute(hosts_tags) unless hosts_tags.empty?
+          hosts_tags.each_slice(MAX_TERMS) do |hosts_tags|
+            prepare(dst, "INSERT INTO hosts_tags (host_id, tag_id) VALUES %s" % hosts_tags.map { "(?, ?)" }.join(", ")).execute(hosts_tags)
+          end
 
           create_index_hosts(dst)
           create_index_tags(dst)

data/lib/hotdog/version.rb

@@ -1,3 +1,3 @@
 module Hotdog
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hotdog
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Yamashita Yuu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-04 00:00:00.000000000 Z
+date: 2015-09-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler