rroonga 7.1.1-x64-mingw32 → 9.0.2-x64-mingw32

data/vendor/local/lib/groonga/scripts/ruby/expression.rb

@@ -1,5 +1,8 @@
 require "expression_rewriter"
 require "expression_rewriters"
+if ENV["GRN_EXPR_OPTIMIZE"] == "yes"
+  require "expression_rewriters/optimizer"
+end
 
 require "expression_tree_builder"
 
@@ -14,7 +17,6 @@ module Groonga
     def rewrite
       rewritten = nil
       begin
-        return nil unless ExpressionRewriters.enabled?
         source = self
         ExpressionRewriters.classes.each do |rewriter_class|
           rewriter = rewriter_class.new(source)

data/vendor/local/lib/groonga/scripts/ruby/expression_rewriters.rb

@@ -7,33 +7,27 @@ module Groonga
         @rewriters[name] = rewriter_class
       end
 
-      def enabled?
-        rewriters_table_name =
-          Config["expression_rewriter.table"] || "expression_rewriters"
-        rewriters_table = Context.instance[rewriters_table_name]
-        return false if rewriters_table.nil?
-        return false if rewriters_table.empty?
-
-        true
-      end
-
       def classes
         rewriters_table_name =
           Config["expression_rewriter.table"] || "expression_rewriters"
         rewriters_table = Context.instance[rewriters_table_name]
-        return [] if rewriters_table.nil?
-
-        rewriters_table.collect do |id|
-          record = Record.new(rewriters_table, id)
-          name = record.key
-          rewriter = @rewriters[name]
-          if rewriter.nil?
-            plugin_name = record.plugin_name.value
-            require plugin_name
+        if rewriters_table.nil?
+          @rewriters.values
+        else
+          rewriters_table.collect do |id|
+            record = Record.new(rewriters_table, id)
+            name = record.key
             rewriter = @rewriters[name]
-            raise "unknown rewriter: <#{name}>:<#{plugin_name}>" if rewriter.nil?
+            if rewriter.nil?
+              plugin_name = record.plugin_name
+              require plugin_name
+              rewriter = @rewriters[name]
+              if rewriter.nil?
+                raise "unknown rewriter: <#{name}>:<#{plugin_name}>"
+              end
+            end
+            rewriter
           end
-          rewriter
         end
       end
     end

data/vendor/local/lib/groonga/scripts/ruby/expression_rewriters/optimizer.rb

@@ -0,0 +1,274 @@
+module Groonga
+  module ExpressionRewriters
+    class Optimizer < ExpressionRewriter
+      register "optimizer"
+
+      def rewrite
+        builder = ExpressionTreeBuilder.new(@expression)
+        root_node = builder.build
+        return @expression if root_node.nil?
+
+        variable = @expression[0]
+        table = context[variable.domain]
+        optimized_root_node = optimize_node(table, root_node)
+
+        rewritten = Expression.create(table)
+        optimized_root_node.build(rewritten)
+        rewritten
+      end
+
+      private
+      def optimize_node(table, node)
+        case node
+        when ExpressionTree::LogicalOperation
+          optimize_and_not(table, node) do |optimized_node|
+            optimized_sub_nodes = optimized_node.nodes.collect do |sub_node|
+              optimize_node(table, sub_node)
+            end
+            case optimized_node.operator
+            when Operator::AND
+              optimized_sub_nodes =
+                optimize_and_sub_nodes(table, optimized_sub_nodes)
+            when Operator::OR
+              optimized_sub_nodes =
+                optimize_or_sub_nodes(table, optimized_sub_nodes)
+            end
+            if optimized_sub_nodes.size == 1
+              optimized_sub_nodes[0]
+            else
+              ExpressionTree::LogicalOperation.new(optimized_node.operator,
+                                                   optimized_sub_nodes)
+            end
+          end
+        when ExpressionTree::BinaryOperation
+          optimize_binary_operation_node(table, node)
+        else
+          node
+        end
+      end
+
+      def optimize_and_not(table, node)
+        return yield(node) unless node.operator == Operator::AND
+
+        optimized_sub_nodes = []
+        sub_nodes = []
+        node.nodes.each do |sub_node|
+          if sub_node.respond_to?(:operator)
+            sub_node_operator = sub_node.operator
+          else
+            sub_node_oeprator = nil
+          end
+          unless sub_node_operator == Operator::NOT
+            sub_nodes << sub_node
+            next
+          end
+
+          case sub_nodes.size
+          when 0
+            sub_nodes << sub_node
+          when 1
+            and_not_node =
+              ExpressionTree::LogicalOperation.new(Operator::AND_NOT,
+                                                   [sub_nodes.first,
+                                                    sub_node.value])
+            optimized_sub_nodes << yield(and_not_node)
+            sub_nodes = []
+          else
+            and_nodes = ExpressionTree::LogicalOperation.new(node.operator,
+                                                             sub_nodes)
+            optimized_and_nodes = yield(and_nodes)
+            and_not_node =
+              ExpressionTree::LogicalOperation.new(Operator::AND_NOT,
+                                                   [optimized_and_nodes,
+                                                    sub_node.value])
+            optimized_sub_nodes << yield(and_not_node)
+            sub_nodes = []
+          end
+        end
+        unless sub_nodes.empty?
+          and_nodes = ExpressionTree::LogicalOperation.new(node.operator,
+                                                           sub_nodes)
+          optimized_sub_nodes << yield(and_nodes)
+        end
+        if optimized_sub_nodes.size == 1
+          optimized_sub_nodes.first
+        else
+          ExpressionTree::LogicalOperation.new(node.operator,
+                                               optimized_sub_nodes)
+        end
+      end
+
+      def optimize_binary_operation_node(table, node)
+        optimized_left = optimize_node(table, node.left)
+        optimized_right = optimize_node(table, node.right)
+        optimized_options = {}
+        if node.parameter
+          optimized_options[:parameter] = optimize_node(table, node.parameter)
+        end
+        optimized_options[:value] = node.value
+        case node.operator
+        when Operator::EQUAL
+          if optimized_left.is_a?(ExpressionTree::Constant) and
+            optimized_right.is_a?(ExpressionTree::Variable)
+            ExpressionTree::BinaryOperation.new(node.operator,
+                                                optimized_right,
+                                                optimized_left,
+                                                optimized_options)
+          else
+            ExpressionTree::BinaryOperation.new(node.operator,
+                                                optimized_left,
+                                                optimized_right,
+                                                optimized_options)
+          end
+        else
+          ExpressionTree::BinaryOperation.new(node.operator,
+                                              optimized_left,
+                                              optimized_right,
+                                              optimized_options)
+        end
+      end
+
+      def group_nodes(nodes)
+        nodes.group_by do |node|
+          case node
+          when ExpressionTree::BinaryOperation
+            if node.left.is_a?(ExpressionTree::Variable)
+              node.left.column
+            else
+              nil
+            end
+          else
+            nil
+          end
+        end
+      end
+
+      def optimize_and_sub_nodes(table, sub_nodes)
+        return sub_nodes unless sub_nodes.all?(&:estimatable?)
+
+        grouped_sub_nodes = group_nodes(sub_nodes)
+
+        optimized_nodes = []
+        grouped_sub_nodes.each do |column, grouped_nodes|
+          if column
+            grouped_nodes = optimize_grouped_and_nodes(column, grouped_nodes)
+          end
+          optimized_nodes.concat(grouped_nodes)
+        end
+
+        optimized_nodes.sort_by do |node|
+          node.estimate_size(table)
+        end
+      end
+
+      COMPARISON_OPERATORS = [
+        Operator::EQUAL,
+        Operator::NOT_EQUAL,
+        Operator::LESS,
+        Operator::GREATER,
+        Operator::LESS_EQUAL,
+        Operator::GREATER_EQUAL,
+      ]
+      def optimize_grouped_and_nodes(column, grouped_nodes)
+        target_nodes, done_nodes = grouped_nodes.partition do |node|
+          node.is_a?(ExpressionTree::BinaryOperation) and
+            COMPARISON_OPERATORS.include?(node.operator) and
+            node.right.is_a?(ExpressionTree::Constant)
+        end
+
+        # TODO: target_nodes = remove_needless_nodes(target_nodes)
+        # e.g.: x < 1 && x < 3 -> x < 1: (x < 3) is meaningless
+
+        if target_nodes.size == 2
+          between_node = try_optimize_between(column, target_nodes)
+          if between_node
+            done_nodes << between_node
+          else
+            done_nodes.concat(target_nodes)
+          end
+        else
+          done_nodes.concat(target_nodes)
+        end
+
+        done_nodes
+      end
+
+      def try_optimize_between(column, target_nodes)
+        greater_node = nil
+        less_node = nil
+        target_nodes.each do |node|
+          case node.operator
+          when Operator::GREATER, Operator::GREATER_EQUAL
+            greater_node = node
+          when Operator::LESS, Operator::LESS_EQUAL
+            less_node = node
+          end
+        end
+        return nil if greater_node.nil? or less_node.nil?
+
+        between = ExpressionTree::Procedure.new(context["between"])
+        if greater_node.operator == Operator::GREATER
+          greater_border = "exclude"
+        else
+          greater_border = "include"
+        end
+        if less_node.operator == Operator::LESS
+          less_border = "exclude"
+        else
+          less_border = "include"
+        end
+        arguments = [
+          ExpressionTree::Variable.new(column),
+          greater_node.right,
+          ExpressionTree::Constant.new(greater_border),
+          less_node.right,
+          ExpressionTree::Constant.new(less_border),
+        ]
+        ExpressionTree::FunctionCall.new(between, arguments)
+      end
+
+      def optimize_or_sub_nodes(table, sub_nodes)
+        grouped_sub_nodes = group_nodes(sub_nodes)
+
+        optimized_nodes = []
+        grouped_sub_nodes.each do |column, grouped_nodes|
+          if column
+            grouped_nodes = optimize_grouped_or_nodes(column, grouped_nodes)
+          end
+          optimized_nodes.concat(grouped_nodes)
+        end
+        optimized_nodes
+      end
+
+      def optimize_grouped_or_nodes(column, grouped_nodes)
+        target_nodes, done_nodes = grouped_nodes.partition do |node|
+          node.is_a?(ExpressionTree::BinaryOperation) and
+            node.operator == Operator::EQUAL and
+            node.right.is_a?(ExpressionTree::Constant)
+        end
+
+        if target_nodes.size > 1
+          in_values_node = try_optimize_in_values(column, target_nodes)
+          if in_values_node
+            done_nodes << in_values_node
+          else
+            done_nodes.concat(target_nodes)
+          end
+        else
+          done_nodes.concat(target_nodes)
+        end
+
+        done_nodes
+      end
+
+      def try_optimize_in_values(column, target_nodes)
+        in_values = ExpressionTree::Procedure.new(context["in_values"])
+        arguments = [ExpressionTree::Variable.new(column)]
+        target_nodes.each do |node|
+          arguments << node.right
+        end
+        ExpressionTree::FunctionCall.new(in_values, arguments)
+      end
+    end
+  end
+end

data/vendor/local/lib/groonga/scripts/ruby/expression_tree.rb

@@ -1,9 +1,15 @@
 require "expression_tree/accessor"
-require "expression_tree/constant"
+require "expression_tree/assign"
+require "expression_tree/assign_binary_operation"
 require "expression_tree/binary_operation"
+require "expression_tree/constant"
 require "expression_tree/function_call"
 require "expression_tree/index_column"
 require "expression_tree/logical_operation"
-require "expression_tree/options"
+require "expression_tree/member"
+require "expression_tree/null"
 require "expression_tree/procedure"
+require "expression_tree/reference"
+require "expression_tree/table"
+require "expression_tree/unary_operation"
 require "expression_tree/variable"

data/vendor/local/lib/groonga/scripts/ruby/expression_tree/assign.rb

@@ -0,0 +1,22 @@
+module Groonga
+  module ExpressionTree
+    class Assign
+      attr_reader :variable
+      attr_reader :value
+      def initialize(variable, value)
+        @variable = variable
+        @value = value
+      end
+
+      def build(expression)
+        @variable.build(expression)
+        @value.build(expression)
+        expression.append_operator(Operator::ASSIGN, 2)
+      end
+
+      def estimatable?
+        false
+      end
+    end
+  end
+end

data/vendor/local/lib/groonga/scripts/ruby/expression_tree/assign_binary_operation.rb

@@ -0,0 +1,24 @@
+module Groonga
+  module ExpressionTree
+    class AssignBinaryOperation
+      attr_reader :operator
+      attr_reader :variable
+      attr_reader :value
+      def initialize(operator, variable, value)
+        @operator = operator
+        @variable = variable
+        @value = value
+      end
+
+      def build(expression)
+        @variable.build(expression)
+        @value.build(expression)
+        expression.append_operator(@operator, 2)
+      end
+
+      def estimatable?
+        false
+      end
+    end
+  end
+end

data/vendor/local/lib/groonga/scripts/ruby/expression_tree/binary_operation.rb

@@ -4,16 +4,33 @@ module Groonga
       attr_reader :operator
       attr_reader :left
       attr_reader :right
-      def initialize(operator, left, right)
+      attr_reader :parameter
+      attr_reader :value
+      def initialize(operator, left, right, options={})
         @operator = operator
         @left = left
         @right = right
+        @parameter = options[:parameter]
+        @value = options[:value]
       end
 
       def build(expression)
         @left.build(expression)
         @right.build(expression)
-        expression.append_operator(@operator, 2)
+        n_args = 2
+        if @parameter
+          @parameter.build(expression)
+          n_args += 1
+        end
+        if @value
+          expression.append_constant(@value, @operator, n_args)
+        else
+          expression.append_operator(@operator, n_args)
+        end
+      end
+
+      def estimatable?
+        true
       end
 
       RANGE_OPERATORS = [
@@ -22,28 +39,116 @@ module Groonga
         Operator::LESS_EQUAL,
         Operator::GREATER_EQUAL,
       ]
+      QUERY_OPERATIONS = [
+        Operator::MATCH,
+        Operator::NEAR,
+        Operator::NEAR2,
+        Operator::SIMILAR,
+        Operator::EXACT,
+        Operator::REGEXP,
+        Operator::FUZZY,
+        Operator::QUORUM,
+      ]
       def estimate_size(table)
         case @operator
+        when Operator::EQUAL
+          estimate_size_equal(table)
         when *RANGE_OPERATORS
           estimate_size_range(table)
+        when *QUERY_OPERATIONS
+          estimate_size_query(table)
+        when Operator::PREFIX
+          estimate_size_prefix(table)
         else
           table.size
         end
       end
 
       private
-      def estimate_size_range(table)
+      def estimate_size_equal(table)
         return table.size unless @left.is_a?(Variable)
         return table.size unless @right.is_a?(Constant)
 
         column = @left.column
         value = @right.value
-        index_info = column.find_index(@operator)
-        return table.size if index_info.nil?
+        case column
+        when Groonga::Accessor
+          return estimate_size_equal_accessor(table, column, value)
+        when Groonga::IndexColumn
+          index_column = column
+        else
+          index_info = column.find_index(@operator)
+          return table.size if index_info.nil?
+          index_column = index_info.index
+        end
+        return table.size unless index_column.respond_to?(:lexicon)
 
-        index_column = index_info.index
         lexicon = index_column.lexicon
-        options = {}
+        term = value.value
+        if value.domain == lexicon.id
+          term_id = term.id
+        else
+          term_id = lexicon[term]
+        end
+        if term_id.nil?
+          0
+        else
+          index_column.estimate_size(term_id: term_id)
+        end
+      end
+
+      def estimate_size_equal_accessor(table, accessor, value)
+        if accessor.id?
+          if table.id?(value.value)
+            return 1
+          else
+            return 0
+          end
+        else
+          if table[value]
+            return 1
+          else
+            return 0
+          end
+        end
+      end
+
+      def estimate_size_range(table)
+        return table.size unless @left.is_a?(Variable)
+        return table.size unless @right.is_a?(Constant)
+
+        is_table_search = false
+        column = @left.column
+        value = @right.value
+        case column
+        when Table
+          is_table_search = true
+          lexicon = column
+        when Groonga::Accessor
+          is_table_search = true
+          lexicon = column.object
+          case lexicon
+          when PatriciaTrie, DoubleArrayTrie
+            # range searchable
+          else
+            return table.size
+          end
+        when Groonga::IndexColumn
+          index_column = column
+          lexicon = column.lexicon
+        else
+          index_info = column.find_index(@operator)
+          return table.size if index_info.nil?
+
+          index_column = index_info.index
+          lexicon = index_column.lexicon
+        end
+        n_terms = lexicon.size
+        return 0 if n_terms.zero?
+
+        options = {
+          limit: sampling_cursor_limit(n_terms),
+        }
         case @operator
         when Operator::LESS
           options[:max] = value
@@ -59,7 +164,100 @@ module Groonga
           options[:flags] = TableCursorFlags::GE
         end
         TableCursor.open(lexicon, options) do |cursor|
-          index_column.estimate_size(:lexicon_cursor => cursor)
+          if is_table_search
+            size = 1
+          else
+            size = index_column.estimate_size(:lexicon_cursor => cursor)
+          end
+          size += 1 if cursor.next != ID::NIL
+          size
+        end
+      end
+
+      def estimate_size_query(table)
+        return table.size unless @left.is_a?(Variable)
+        return table.size unless @right.is_a?(Constant)
+
+        column = @left.column
+        indexes = []
+        case column
+        when Expression
+          column.codes.each do |code|
+            case code.op
+            when Operator::GET_VALUE
+              value = code.value
+              case value
+              when IndexColumn
+                indexes << index
+              when Column
+                index_info = value.find_index(@operator)
+                return table.size if index_info.nil?
+                indexes << index_info.index
+              end
+            end
+          end
+        when Groonga::IndexColumn
+          indexes << column
+        else
+          index_info = column.find_index(@operator)
+          return table.size if index_info.nil?
+          indexes << index_info.index
+        end
+
+        return table.size if indexes.empty?
+
+        value = @right.value
+        sizes = indexes.collect do |index|
+          index.estimate_size(:query => value,
+                              :mode => @operator)
+        end
+        sizes.max
+      end
+
+      def estimate_size_prefix(table)
+        return table.size unless @left.is_a?(Variable)
+        return table.size unless @right.is_a?(Constant)
+
+        is_table_search = false
+        column = @left.column
+        case column
+        when Groonga::Accessor
+          return table.size unless column.key?
+          is_table_search = true
+          lexicon = column.object
+        when Groonga::IndexColumn
+          lexicon = column.lexicon
+        else
+          return table.size
+        end
+        n_terms = lexicon.size
+        return 0 if n_terms.zero?
+
+        value = @right.value
+        options = {
+          min: value,
+          limit: sampling_cursor_limit(n_terms),
+          flags: TableCursorFlags::PREFIX,
+        }
+        TableCursor.open(lexicon, options) do |cursor|
+          if is_table_search
+            size = 1
+          else
+            size = column.estimate_size(:lexicon_cursor => cursor)
+          end
+          size += 1 if cursor.next != ID::NIL
+          size
+        end
+      end
+
+      def sampling_cursor_limit(n_terms)
+        limit = n_terms * 0.01
+        if limit < 10
+          10
+        elsif limit > 1000
+          1000
+        else
+          limit.to_i
         end
       end
     end