nose 0.1.0pre

data/lib/nose/search/problem.rb

@@ -0,0 +1,328 @@
+# frozen_string_literal: true
+
+require 'logging'
+
+begin
+  require 'mipper'
+rescue LoadError
+  # We can't use most search functionality, but it won't explode
+  nil
+end
+
+module NoSE
+  module Search
+    # Simple enum for possible objective functions
+    module Objective
+      # Minimize the cost of statements in the workload
+      COST  = 1
+
+      # Minimize the space usage of generated indexes
+      SPACE = 2
+
+      # Minimize the total number of indexes
+      INDEXES = 3
+    end
+
+    # A representation of a search problem as an ILP
+    class Problem
+      attr_reader :model, :status, :queries, :updates,
+                  :index_vars, :query_vars, :indexes, :data,
+                  :objective_type, :objective_value
+
+      def initialize(queries, updates, data, objective = Objective::COST)
+        @queries = queries
+        @updates = updates
+        @data = data
+        @indexes = @data[:costs].flat_map { |_, ic| ic.keys }.uniq
+        @logger = Logging.logger['nose::search::problem']
+        @status = nil
+        @objective_type = objective
+
+        setup_model
+      end
+
+      # Run the solver and make the selected indexes available
+      # @return [void]
+      def solve(previous_type = nil)
+        return unless @status.nil?
+
+        # Run the optimization
+        @model.optimize
+        @status = model.status
+        fail NoSolutionException, @status if @status != :optimized
+
+        # Store the objective value
+        @objective_value = @obj_var.value
+
+        if @objective_type != Objective::INDEXES && previous_type.nil?
+          solve_next Objective::INDEXES
+          return
+        elsif !previous_type.nil? && previous_type != Objective::SPACE
+          solve_next Objective::SPACE
+          return
+        elsif @objective_value.nil?
+          @objective_value = @model.objective_value
+        end
+
+        @logger.debug do
+          "Final objective value is #{@objective.inspect}" \
+            " = #{@objective_value}"
+        end
+      end
+
+      # Return the selected indices
+      # @return [Set<Index>]
+      def selected_indexes
+        return if @status.nil?
+        return @selected_indexes if @selected_indexes
+
+        @selected_indexes = @index_vars.each_key.select do |index|
+          @index_vars[index].value
+        end.to_set
+      end
+
+      # Return relevant data on the results of the ILP
+      # @return [Results]
+      def result
+        result = Results.new self, @data[:by_id_graph]
+        result.enumerated_indexes = indexes
+        result.indexes = selected_indexes
+
+        # TODO: Update for indexes grouped by ID path
+        result.total_size = selected_indexes.sum_by(&:size)
+        result.total_cost = @objective_value
+
+        result
+      end
+
+      # Get the size of all indexes in the workload
+      # @return [MIPPeR::LinExpr]
+      def total_size
+        # TODO: Update for indexes grouped by ID path
+        @indexes.map do |index|
+          @index_vars[index] * (index.size * 1.0)
+        end.reduce(&:+)
+      end
+
+      # Get the cost of all queries in the workload
+      # @return [MIPPeR::LinExpr]
+      def total_cost
+        cost = @queries.reduce(MIPPeR::LinExpr.new) do |expr, query|
+          expr.add(@indexes.reduce(MIPPeR::LinExpr.new) do |subexpr, index|
+            subexpr.add total_query_cost(@data[:costs][query][index],
+                                         @query_vars[index][query],
+                                         @sort_costs[query][index],
+                                         @sort_vars[query][index])
+          end)
+        end
+
+        cost = add_update_costs cost
+        cost
+      end
+
+      # The total number of indexes
+      # @return [MIPPeR::LinExpr]
+      def total_indexes
+        total = MIPPeR::LinExpr.new
+        @index_vars.each_value { |var| total += var * 1.0 }
+
+        total
+      end
+
+      private
+
+      # Pin the current objective value and set a new objective
+      # @return [void]
+      def solve_next(objective_type)
+        @obj_var.lower_bound = @objective_value
+        @obj_var.upper_bound = @objective_value
+
+        if objective_type == Objective::INDEXES
+          @objective_type = Objective::INDEXES
+          define_objective 'objective_indexes'
+        elsif objective_type == Objective::SPACE
+          @objective_type = Objective::SPACE
+          define_objective 'objective_space'
+        end
+
+        @status = nil
+        solve objective_type
+      end
+
+      # Write a model to a temporary file and log the file name
+      # @return [void]
+      def log_model(type)
+        @logger.debug do
+          tmpfile = Tempfile.new ['model', '.mps']
+          ObjectSpace.undefine_finalizer tmpfile
+          @model.write_mps tmpfile.path
+          "#{type} written to #{tmpfile.path}"
+        end
+      end
+
+      # Build the ILP by creating all the variables and constraints
+      # @return [void]
+      def setup_model
+        # Set up solver environment
+        @model = MIPPeR::CbcModel.new
+
+        add_variables
+        prepare_sort_costs
+        @model.update
+
+        add_constraints
+        define_objective
+        @model.update
+
+        log_model 'Model'
+      end
+
+      private
+
+      # Set the value of the objective function (workload cost)
+      # @return [void]
+      def define_objective(var_name = 'objective')
+        obj = case @objective_type
+              when Objective::COST
+                total_cost
+              when Objective::SPACE
+                total_size
+              when Objective::INDEXES
+                total_indexes
+              end
+
+        # Add the objective function as a variable
+        var_name = nil unless ENV['NOSE_LOG'] == 'debug'
+        @obj_var = MIPPeR::Variable.new 0, Float::INFINITY, 1.0,
+                                        :continuous, var_name
+        @model << @obj_var
+        @model.update
+
+        @model << MIPPeR::Constraint.new(obj + @obj_var * -1.0, :==, 0.0)
+
+        @logger.debug { "Objective function is #{obj.inspect}" }
+
+        @objective = obj
+        @model.sense = :min
+      end
+
+      # Initialize query and index variables
+      # @return [void]
+      def add_variables
+        @index_vars = {}
+        @query_vars = {}
+        @indexes.each do |index|
+          @query_vars[index] = {}
+          @queries.each_with_index do |query, q|
+            query_var = "q#{q}_#{index.key}" if ENV['NOSE_LOG'] == 'debug'
+            var = MIPPeR::Variable.new 0, 1, 0, :binary, query_var
+            @model << var
+            @query_vars[index][query] = var
+          end
+
+          var_name = index.key if ENV['NOSE_LOG'] == 'debug'
+          @index_vars[index] = MIPPeR::Variable.new 0, 1, 0, :binary, var_name
+
+          # If needed when grouping by ID graph, add an extra
+          # variable for the base index based on the ID graph
+          next unless @data[:by_id_graph]
+          id_graph = index.to_id_graph
+          next if id_graph == index
+
+          # Add a new variable for the ID graph if needed
+          unless @index_vars.key? id_graph
+            var_name = index.key if ENV['NOSE_LOG'] == 'debug'
+            @index_vars[id_graph] = MIPPeR::Variable.new 0, 1, 0, :binary,
+                                                         var_name
+          end
+
+          # Ensure that the ID graph of this index is present if we use it
+          name = "ID_#{id_graph.key}_#{index.key}" \
+            if ENV['NOSE_LOG'] == 'debug'
+          constr = MIPPeR::Constraint.new @index_vars[id_graph] * 1.0 + \
+                                          @index_vars[index] * -1.0,
+                                          :>=, 0, name
+          @model << constr
+        end
+
+        @index_vars.each_value { |var| @model << var }
+      end
+
+      # Prepare variables and constraints to account for the cost of sorting
+      # @return [void]
+      def prepare_sort_costs
+        @sort_costs = {}
+        @sort_vars = {}
+        @data[:costs].each do |query, index_costs|
+          @sort_costs[query] = {}
+          @sort_vars[query] = {}
+
+          index_costs.each do |index, (steps, _)|
+            sort_step = steps.find { |s| s.is_a?(Plans::SortPlanStep) }
+            next if sort_step.nil?
+
+            @sort_costs[query][index] ||= sort_step.cost
+            q = @queries.index query
+
+            name = "s#{q}" if ENV['NOSE_LOG'] == 'debug'
+            sort_var = MIPPeR::Variable.new 0, 1, 0, :binary, name
+            @sort_vars[query][index] ||= sort_var
+            @model << sort_var
+
+            name = "q#{q}_#{index.key}_sort" if ENV['NOSE_LOG'] == 'debug'
+            constr = MIPPeR::Constraint.new @sort_vars[query][index] * 1.0 +
+                                            @query_vars[index][query] * -1.0,
+                                            :>=, 0, name
+            @model << constr
+          end
+        end
+      end
+
+      # Add all necessary constraints to the model
+      # @return [void]
+      def add_constraints
+        [
+          IndexPresenceConstraints,
+          SpaceConstraint,
+          CompletePlanConstraints
+        ].each { |constraint| constraint.apply self }
+
+        @logger.debug do
+          "Added #{@model.constraints.count} constraints to model"
+        end
+      end
+
+      # Deal with updates which do not require support queries
+      # @return [MIPPeR::LinExpr]
+      def add_update_costs(min_cost)
+        @updates.each do |update|
+          @indexes.each do |index|
+            index = index.to_id_graph if data[:by_id_graph]
+            next unless update.modifies_index?(index)
+
+            min_cost.add @index_vars[index] *
+                         @data[:update_costs][update][index]
+          end
+        end
+
+        min_cost
+      end
+
+      # Get the total cost of the query for the objective function
+      # @return [MIPPeR::LinExpr]
+      def total_query_cost(cost, query_var, sort_cost, sort_var)
+        return MIPPeR::LinExpr.new if cost.nil?
+        query_cost = cost.last * 1.0
+
+        cost_expr = query_var * query_cost
+        cost_expr += sort_var * sort_cost unless sort_cost.nil?
+
+        cost_expr
+      end
+    end
+
+    # Thrown when no solution can be found to the ILP
+    class NoSolutionException < StandardError
+    end
+  end
+end

data/lib/nose/search/results.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module NoSE
+  module Search
+    # A container for results from a schema search
+    class Results
+      attr_reader :cost_model
+      attr_accessor :enumerated_indexes, :indexes, :total_size, :total_cost,
+                    :workload, :update_plans, :plans,
+                    :revision, :time, :command, :by_id_graph
+
+      def initialize(problem = nil, by_id_graph = false)
+        @problem = problem
+        return if problem.nil?
+        @by_id_graph = by_id_graph
+
+        # Find the indexes the ILP says the query should use
+        @query_indexes = Hash.new { |h, k| h[k] = Set.new }
+        @problem.query_vars.each do |index, query_vars|
+          query_vars.each do |query, var|
+            next unless var.value
+            @query_indexes[query].add index
+          end
+        end
+      end
+
+      # Provide access to the underlying model in the workload
+      # @return [Model]
+      def model
+        @workload.nil? ? @model : @workload.model
+      end
+
+      # Assign the model to the workload if it exists, otherwise store it
+      # @return [void]
+      def model=(model)
+        if @workload.nil?
+          @model = model
+        else
+          @workload.instance_variable_set :@model, model
+        end
+      end
+
+      # After setting the cost model, recalculate the cost
+      # @return [void]
+      def cost_model=(new_cost_model)
+        recalculate_cost new_cost_model
+        @cost_model = new_cost_model
+      end
+
+      # After setting the cost model, recalculate the cost
+      # @return [void]
+      def recalculate_cost(new_cost_model = nil)
+        new_cost_model = @cost_model if new_cost_model.nil?
+
+        (@plans || []).each do |plan|
+          plan.each { |s| s.calculate_cost new_cost_model }
+        end
+        (@update_plans || []).each do |plan|
+          plan.update_steps.each { |s| s.calculate_cost new_cost_model }
+          plan.query_plans.each do |query_plan|
+            query_plan.each { |s| s.calculate_cost new_cost_model }
+          end
+        end
+
+        # Recalculate the total
+        query_cost = (@plans || []).sum_by do |plan|
+          plan.cost * @workload.statement_weights[plan.query]
+        end
+        update_cost = (@update_plans || []).sum_by do |plan|
+          plan.cost * @workload.statement_weights[plan.statement]
+        end
+        @total_cost = query_cost + update_cost
+      end
+
+      # Validate that the results of the search are consistent
+      # @return [void]
+      def validate
+        validate_indexes
+        validate_query_indexes @plans
+        validate_update_indexes
+
+        planned_queries = plans.map(&:query).to_set
+        fail InvalidResultsException unless \
+          (@workload.queries.to_set - planned_queries).empty?
+        validate_query_plans @plans
+
+        validate_update_plans
+        validate_objective
+
+        freeze
+      end
+
+      # Set the query plans which should be used based on the entire tree
+      # @return [void]
+      def plans_from_trees(trees)
+        @plans = trees.map do |tree|
+          # Exclude support queries since they will be in update plans
+          query = tree.query
+          next if query.is_a?(SupportQuery)
+
+          select_plan tree
+        end.compact
+      end
+
+      # Select the single query plan from a tree of plans
+      # @return [Plans::QueryPlan]
+      # @raise [InvalidResultsException]
+      def select_plan(tree)
+        query = tree.query
+        plan = tree.find do |tree_plan|
+          tree_plan.indexes.to_set == @query_indexes[query]
+        end
+        plan.instance_variable_set :@workload, @workload
+
+        fail InvalidResultsException if plan.nil?
+        plan
+      end
+
+      private
+
+      # Check that the indexes selected were actually enumerated
+      # @return [void]
+      def validate_indexes
+        # We may not have enumerated ID graphs
+        check_indexes = @indexes.dup
+        @indexes.each do |index|
+          check_indexes.delete index.to_id_graph
+        end if @by_id_graph
+
+        fail InvalidResultsException unless \
+          (check_indexes - @enumerated_indexes).empty?
+      end
+
+      # Ensure we only have necessary update plans which use available indexes
+      # @return [void]
+      def validate_update_indexes
+        @update_plans.each do |plan|
+          validate_query_indexes plan.query_plans
+          valid_plan = @indexes.include?(plan.index)
+          fail InvalidResultsException unless valid_plan
+        end
+      end
+
+      # Check that the objective function has the expected value
+      # @return [void]
+      def validate_objective
+        if @problem.objective_type == Objective::COST
+          query_cost = @plans.reduce 0 do |sum, plan|
+            sum + @workload.statement_weights[plan.query] * plan.cost
+          end
+          update_cost = @update_plans.reduce 0 do |sum, plan|
+            sum + @workload.statement_weights[plan.statement] * plan.cost
+          end
+          cost = query_cost + update_cost
+
+          fail InvalidResultsException unless (cost - @total_cost).abs < 0.001
+        elsif @problem.objective_type == Objective::SPACE
+          size = @indexes.sum_by(&:size)
+          fail InvalidResultsException unless (size - @total_size).abs < 0.001
+        end
+      end
+
+      # Ensure that all the query plans use valid indexes
+      # @return [void]
+      def validate_query_indexes(plans)
+        plans.each do |plan|
+          plan.each do |step|
+            valid_plan = !step.is_a?(Plans::IndexLookupPlanStep) ||
+                         @indexes.include?(step.index)
+            fail InvalidResultsException unless valid_plan
+          end
+        end
+      end
+
+      # Validate the query plans from the original workload
+      # @return [void]
+      def validate_query_plans(plans)
+        # Check that these indexes are actually used by the query
+        plans.each do |plan|
+          fail InvalidResultsException unless \
+            plan.indexes.to_set == @query_indexes[plan.query]
+        end
+      end
+
+      # Validate the support query plans for each update
+      # @return [void]
+      def validate_update_plans
+        @update_plans.each do |plan|
+          plan.instance_variable_set :@workload, @workload
+
+          validate_query_plans plan.query_plans
+        end
+      end
+    end
+
+    # Thrown when a search produces invalid results
+    class InvalidResultsException < StandardError
+    end
+  end
+end