nose 0.1.0pre

data/lib/nose/plans/execution_plan.rb

@@ -0,0 +1,282 @@
+# frozen_string_literal: true
+
+module NoSE
+  module Plans
+    # Simple DSL for constructing execution plans
+    class ExecutionPlans
+      # The subdirectory execution plans are loaded from
+      LOAD_PATH = 'plans'
+      include Loader
+
+      attr_reader :groups, :weights, :schema, :mix
+
+      def initialize(&block)
+        @groups = Hash.new { |h, k| h[k] = [] }
+        @weights = Hash.new { |h, k| h[k] = {} }
+        @mix = :default
+
+        instance_eval(&block) if block_given?
+
+        # Reset the mix to force weight assignment
+        self.mix = @mix
+      end
+
+      # Populate the cost of each plan
+      # @return [void]
+      def calculate_cost(cost_model)
+        @groups.each_value do |plans|
+          plans.each do |plan|
+            plan.steps.each do |step|
+              cost = cost_model.index_lookup_cost step
+              step.instance_variable_set(:@cost, cost)
+            end
+
+            plan.query_plans.each do |query_plan|
+              query_plan.steps.each do |step|
+                cost = cost_model.index_lookup_cost step
+                step.instance_variable_set(:@cost, cost)
+              end
+            end
+
+            # XXX Only bother with insert statements for now
+            plan.update_steps.each do |step|
+              cost = cost_model.insert_cost step
+              step.instance_variable_set(:@cost, cost)
+            end
+          end
+        end
+      end
+
+      # Set the weights on plans when the mix is changed
+      # @return [void]
+      def mix=(mix)
+        @mix = mix
+
+        @groups.each do |group, plans|
+          plans.each do |plan|
+            plan.instance_variable_set :@weight, @weights[group][@mix]
+            plan.query_plans.each do |query_plan|
+              query_plan.instance_variable_set :@weight, @weights[group][@mix]
+            end
+          end
+        end
+      end
+
+      # rubocop:disable MethodName
+
+      # Set the schema to be used by the execution plans
+      # @return [void]
+      def Schema(name)
+        @schema = Schema.load name
+        NoSE::DSL.mixin_fields @schema.model.entities, QueryExecutionPlan
+        NoSE::DSL.mixin_fields @schema.model.entities, ExecutionPlans
+      end
+
+      # Set the default mix for these plans
+      # @return [void]
+      def DefaultMix(mix)
+        self.mix = mix
+      end
+
+      # Define a group of query execution plans
+      # @return [void]
+      def Group(name, weight = 1.0, **mixes, &block)
+        @group = name
+
+        # Save the weights
+        if mixes.empty?
+          @weights[name][:default] = weight
+        else
+          @weights[name] = mixes
+        end
+
+        instance_eval(&block) if block_given?
+      end
+
+      # Define a single plan within a group
+      # @return [void]
+      def Plan(name, &block)
+        return unless block_given?
+
+        plan = QueryExecutionPlan.new(@group, name, @schema, self)
+
+        # Capture one level of nesting in plans
+        if @parent_plan.nil?
+          @parent_plan = plan if @parent_plan.nil?
+          set_parent = true
+        else
+          set_parent = false
+        end
+
+        plan.instance_eval(&block)
+
+        # Reset the parent plan if it was set
+        @parent_plan = nil if set_parent
+
+        @groups[@group] << plan
+      end
+
+      # Add support queries for updates in a plan
+      # @return [void]
+      def Support(&block)
+        # XXX Hack to swap the group name and capture support plans
+        old_group = @group
+        @group = '__SUPPORT__'
+        instance_eval(&block) if block_given?
+
+        @parent_plan.query_plans = @groups[@group]
+        @parent_plan.query_plans.each do |plan|
+          plan.instance_variable_set(:@group, old_group)
+        end
+
+        @groups[@group] = []
+
+        @group = old_group
+      end
+
+      # rubocop:enable MethodName
+    end
+
+    # DSL to construct query execution plans
+    class QueryExecutionPlan < AbstractPlan
+      attr_reader :group, :name, :params, :select_fields,
+                  :steps, :update_steps, :index
+      attr_accessor :query_plans
+
+      # Most of the work is delegated to the array
+      extend Forwardable
+      def_delegators :@steps, :each, :<<, :[], :==, :===, :eql?,
+                     :inspect, :to_s, :to_a, :to_ary, :last, :length, :count
+
+      def initialize(group, name, schema, plans)
+        @group = group
+        @name = name
+        @schema = schema
+        @plans = plans
+        @select_fields = []
+        @params = {}
+        @steps = []
+        @update_steps = []
+        @query_plans = []
+      end
+
+      # Produce the fields updated by this plan
+      # @return [Array<Fields::Field>]
+      def update_fields
+        @update_steps.last.fields
+      end
+
+      # These plans have no associated query
+      # @return [nil]
+      def query
+        nil
+      end
+
+      # The estimated cost of executing this plan
+      # @return [Fixnum]
+      def cost
+        costs = @steps.map(&:cost) + @update_steps.map(&:cost)
+        costs += @query_plans.map(&:steps).flatten.map(&:cost)
+
+        costs.inject(0, &:+)
+      end
+
+      # rubocop:disable MethodName
+
+      # Identify fields to be selected
+      # @return [void]
+      def Select(*fields)
+        @select_fields = fields.flatten.to_set
+      end
+
+      # Add parameters which are used as input to the plan
+      # @return [void]
+      def Param(field, operator, value = nil)
+        operator = :'=' if operator == :==
+        @params[field.id] = Condition.new(field, operator, value)
+      end
+
+      # Pass the support query up to the parent
+      def Support(&block)
+        @plans.Support(&block)
+      end
+
+      # Create a new index lookup step with a particular set of conditions
+      # @return [void]
+      def Lookup(index_key, *conditions, limit: nil)
+        index = @schema.indexes[index_key]
+
+        step = Plans::IndexLookupPlanStep.new index
+        eq_fields = Set.new
+        range_field = nil
+        conditions.each do |field, operator|
+          if operator == :==
+            eq_fields.add field
+          else
+            range_field = field
+          end
+        end
+
+        step.instance_variable_set :@eq_filter, eq_fields
+        step.instance_variable_set :@range_filter, range_field
+
+        # XXX No ordering supported for now
+        step.instance_variable_set :@order_by, []
+
+        step.instance_variable_set :@limit, limit unless limit.nil?
+
+        # Cardinality calculations adapted from
+        # IndexLookupPlanStep#update_state
+        state = OpenStruct.new
+        if @steps.empty?
+          state.hash_cardinality = 1
+        else
+          state.hash_cardinality = @steps.last.state.cardinality
+        end
+        cardinality = index.per_hash_count * state.hash_cardinality
+        state.cardinality = Cardinality.filter cardinality,
+                                               eq_fields - index.hash_fields,
+                                               range_field
+
+        step.state = state
+
+        @steps << step
+      end
+
+      # Add a new insertion step into an index
+      # @return [void]
+      def Insert(index_key, *fields)
+        @index = @schema.indexes[index_key]
+
+        # Get cardinality from last step of each support query plan
+        # as in UpdatePlanner#find_plans_for_update
+        cardinalities = @query_plans.map { |p| p.steps.last.state.cardinality }
+        cardinality = cardinalities.inject(1, &:*)
+        state = OpenStruct.new cardinality: cardinality
+
+        fields = @index.all_fields if fields.empty?
+        step = Plans::InsertPlanStep.new @index, state, fields
+
+        @update_steps << step
+      end
+
+      # Add a new deletion step from an index
+      # @return [void]
+      def Delete(index_key)
+        @index = @schema.indexes[index_key]
+
+        step = Plans::DeletePlanStep.new @index
+
+        # Get cardinality from last step of each support query plan
+        # as in UpdatePlanner#find_plans_for_update
+        cardinalities = @query_plans.map { |p| p.steps.last.state.cardinality }
+        cardinality = cardinalities.inject(1, &:*)
+        step.state = OpenStruct.new cardinality: cardinality
+
+        @update_steps << step
+      end
+
+      # rubocop:enable MethodName
+    end
+  end
+end

data/lib/nose/plans/filter.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module NoSE
+  module Plans
+    # A query plan performing a filter without an index
+    class FilterPlanStep < PlanStep
+      attr_reader :eq, :range
+
+      def initialize(eq, range, state = nil)
+        @eq = eq
+        @range = range
+        super()
+
+        return if state.nil?
+        @state = state.dup
+        update_state
+        @state.freeze
+      end
+
+      # Two filtering steps are equal if they filter on the same fields
+      # @return [Boolean]
+      def ==(other)
+        other.instance_of?(self.class) && \
+          @eq == other.eq && @range == other.range
+      end
+
+      def hash
+        [@eq.map(&:id), @range.nil? ? nil : @range.id].hash
+      end
+
+      # :nocov:
+      def to_color
+        "#{super} #{@eq.to_color} #{@range.to_color} " +
+          begin
+            "#{@parent.state.cardinality} " \
+              "-> #{state.cardinality}"
+          rescue NoMethodError
+            ''
+          end
+      end
+      # :nocov:
+
+      # Check if filtering can be done (we have all the necessary fields)
+      def self.apply(parent, state)
+        # Get fields and check for possible filtering
+        filter_fields, eq_filter, range_filter = filter_fields parent, state
+        return nil if filter_fields.empty?
+
+        FilterPlanStep.new eq_filter, range_filter, state \
+          if required_fields?(filter_fields, parent)
+      end
+
+      # Get the fields we can possibly filter on
+      def self.filter_fields(parent, state)
+        eq_filter = state.eq.select { |field| parent.fields.include? field }
+        filter_fields = eq_filter.dup
+        if state.range && parent.fields.include?(state.range)
+          range_filter = state.range
+          filter_fields << range_filter
+        else
+          range_filter = nil
+        end
+
+        [filter_fields, eq_filter, range_filter]
+      end
+      private_class_method :filter_fields
+
+      # Check that we have all the fields we are filtering
+      # @return [Boolean]
+      def self.required_fields?(filter_fields, parent)
+        filter_fields.map do |field|
+          next true if parent.fields.member? field
+
+          # We can also filter if we have a foreign key
+          # XXX for now we assume this value is the same
+          next unless field.is_a? IDField
+          parent.fields.any? do |pfield|
+            pfield.is_a?(ForeignKeyField) && pfield.entity == field.parent
+          end
+        end.all?
+      end
+      private_class_method :required_fields?
+
+      private
+
+      # Apply the filters and perform a uniform estimate on the cardinality
+      # @return [void]
+      def update_state
+        @state.eq -= @eq
+        @state.cardinality *= @eq.map { |field| 1.0 / field.cardinality } \
+                                 .inject(1.0, &:*)
+        return unless @range
+
+        @state.range = nil
+        @state.cardinality *= 0.1
+      end
+    end
+  end
+end

data/lib/nose/plans/index_lookup.rb

@@ -0,0 +1,302 @@
+# frozen_string_literal: true
+
+module NoSE
+  module Plans
+    # Superclass for steps using indices
+    class IndexLookupPlanStep < PlanStep
+      extend Forwardable
+
+      attr_reader :index, :eq_filter, :range_filter, :limit, :order_by
+      delegate hash: :index
+
+      def initialize(index, state = nil, parent = nil)
+        super()
+        @index = index
+
+        if state && state.query
+          all_fields = state.query.all_fields
+          @fields = (@index.hash_fields + @index.order_fields).to_set + \
+                    (@index.extra.to_set & all_fields)
+        else
+          @fields = @index.all_fields
+        end
+
+        return if state.nil?
+        @state = state.dup
+        update_state parent
+        @state.freeze
+      end
+
+      # :nocov:
+      def to_color
+        if @state.nil?
+          "#{super} #{@index.to_color}"
+        else
+          "#{super} #{@index.to_color} * " \
+            "#{@state.cardinality}/#{@state.hash_cardinality} "
+        end
+      end
+      # :nocov:
+
+      # Two index steps are equal if they use the same index
+      def ==(other)
+        other.instance_of?(self.class) && @index == other.index
+      end
+      alias eql? ==
+
+      # Check if this step can be applied for the given index,
+      # returning a possible application of the step
+      # @return [IndexLookupPlanStep]
+      def self.apply(parent, index, state)
+        # Validate several conditions which identify if this index is usable
+        begin
+          check_joins index, state
+          check_forward_lookup parent, index, state
+          check_parent_index parent, index, state
+          check_all_hash_fields parent, index, state
+          check_graph_fields parent, index, state
+          check_last_fields index, state
+        rescue InvalidIndex
+          return nil
+        end
+
+        IndexLookupPlanStep.new(index, state, parent)
+      end
+
+      # Check that this index is a valid continuation of the set of joins
+      # @raise [InvalidIndex]
+      # @return [void]
+      def self.check_joins(index, state)
+        fail InvalidIndex \
+          unless index.graph.entities.include?(state.joins.first) &&
+                 (index.graph.unique_edges &
+                  state.graph.unique_edges ==
+                  index.graph.unique_edges)
+      end
+      private_class_method :check_joins
+
+      # Check that this index moves forward on the list of joins
+      # @raise [InvalidIndex]
+      # @return [void]
+      def self.check_forward_lookup(parent, index, state)
+        # XXX This disallows plans which look up additional attributes
+        #     for entities other than the final one
+        fail InvalidIndex if index.graph.size == 1 && state.graph.size > 1 &&
+                             !parent.is_a?(RootPlanStep)
+        fail InvalidIndex if index.identity? && state.graph.size > 1
+      end
+      private_class_method :check_forward_lookup
+
+      # Check if this index can be used after the current parent
+      # @return [Boolean]
+      def self.invalid_parent_index?(state, index, parent_index)
+        return false if parent_index.nil?
+
+        # We don't do multiple lookups by ID for the same entity set
+        return true if parent_index.identity? &&
+                       index.graph == parent_index.graph
+
+        # If the last step gave an ID, we must use it
+        # XXX This doesn't cover all cases
+        last_parent_entity = state.joins.reverse.find do |entity|
+          parent_index.graph.entities.include? entity
+        end
+        parent_ids = Set.new [last_parent_entity.id_field]
+        has_ids = parent_ids.subset? parent_index.all_fields
+        return true if has_ids && index.hash_fields.to_set != parent_ids
+
+        # If we're looking up from a previous step, only allow lookup by ID
+        return true unless (index.graph.size == 1 &&
+                           parent_index.graph != index.graph) ||
+                           index.hash_fields == parent_ids
+      end
+      private_class_method :invalid_parent_index?
+
+      # Check that this index is a valid continuation of the set of joins
+      # @raise [InvalidIndex]
+      # @return [void]
+      def self.check_parent_index(parent, index, state)
+        fail InvalidIndex \
+          if invalid_parent_index? state, index, parent.parent_index
+      end
+      private_class_method :check_parent_index
+
+      # Check that we have all hash fields needed to perform the lookup
+      # @raise [InvalidIndex]
+      # @return [void]
+      def self.check_all_hash_fields(parent, index, state)
+        fail InvalidIndex unless index.hash_fields.all? do |field|
+          (parent.fields + state.given_fields).include? field
+        end
+      end
+      private_class_method :check_all_hash_fields
+
+      # Get fields in the query relevant to this index
+      # and check that they are provided for us here
+      # @raise [InvalidIndex]
+      # @return [void]
+      def self.check_graph_fields(parent, index, state)
+        hash_entity = index.hash_fields.first.parent
+        graph_fields = state.fields_for_graph(index.graph, hash_entity).to_set
+        graph_fields -= parent.fields # exclude fields already fetched
+        fail InvalidIndex unless graph_fields.subset?(index.all_fields)
+      end
+      private_class_method :check_graph_fields
+
+      # Check that we have the required fields to move on with the next lookup
+      # @return [Boolean]
+      def self.last_fields?(index, state)
+        index_includes = lambda do |fields|
+          fields.all? { |f| index.all_fields.include? f }
+        end
+
+        # We must have either the ID or all the fields
+        # for leaf entities in the original graph
+        leaf_entities = index.graph.entities.select do |entity|
+          state.graph.leaf_entity?(entity)
+        end
+        leaf_entities.all? do |entity|
+          index_includes.call([entity.id_field]) ||
+            index_includes.call(state.fields.select { |f| f.parent == entity })
+        end
+      end
+      private_class_method :last_fields?
+
+      # @raise [InvalidIndex]
+      # @return [void]
+      def self.check_last_fields(index, state)
+        fail InvalidIndex unless last_fields?(index, state)
+      end
+      private_class_method :check_last_fields
+
+      private
+
+      # Get the set of fields which can be filtered by the ordered keys
+      # @return [Array<Fields::Field>]
+      def range_order_prefix
+        order_prefix = (@state.eq - @index.hash_fields) & @index.order_fields
+        order_prefix << @state.range unless @state.range.nil?
+        order_prefix = order_prefix.zip(@index.order_fields)
+        order_prefix.take_while { |x, y| x == y }.map(&:first)
+      end
+
+      # Perform any ordering implicit to this index
+      # @return [Boolean] whether this index is by ID
+      def resolve_order
+        # We can't resolve ordering if we're doing an ID lookup
+        # since only one record exists per row (if it's the same entity)
+        # We also need to have the fields used in order
+        first_join = @state.query.join_order.detect do |entity|
+          @index.graph.entities.include? entity
+        end
+        indexed_by_id = @index.hash_fields.include?(first_join.id_field)
+        order_prefix = @state.order_by.longest_common_prefix(
+          @index.order_fields - @eq_filter.to_a
+        )
+        if indexed_by_id && order_prefix.map(&:parent).to_set ==
+                            Set.new([@index.hash_fields.first.parent])
+          order_prefix = []
+        else
+          @state.order_by -= order_prefix
+        end
+        @order_by = order_prefix
+
+        indexed_by_id
+      end
+
+      # Strip the graph for this index, but if we haven't fetched all
+      # fields, leave the last one so we can perform a separate ID lookup
+      # @return [void]
+      def strip_graph
+        hash_entity = @index.hash_fields.first.parent
+        @state.graph = @state.graph.dup
+        required_fields = @state.fields_for_graph(@index.graph, hash_entity,
+                                                  select: true).to_set
+        if required_fields.subset?(@index.all_fields) &&
+           @state.graph == @index.graph
+          removed_nodes = @state.joins[0..@index.graph.size]
+          @state.joins = @state.joins[@index.graph.size..-1]
+        else
+          removed_nodes = if index.graph.size == 1
+                            []
+                          else
+                            @state.joins[0..@index.graph.size - 2]
+                          end
+          @state.joins = @state.joins[@index.graph.size - 1..-1]
+        end
+
+        # Remove nodes which have been processed from the graph
+        @state.graph.remove_nodes removed_nodes
+      end
+
+      # Update the cardinality of this step, applying a limit if possible
+      def update_cardinality(parent, indexed_by_id)
+        # Calculate the new cardinality assuming no limit
+        # Hash cardinality starts at 1 or is the previous cardinality
+        if parent.is_a?(RootPlanStep)
+          @state.hash_cardinality = 1
+        else
+          @state.hash_cardinality = parent.state.cardinality
+        end
+
+        # Filter the total number of rows by filtering on non-hash fields
+        cardinality = @index.per_hash_count * @state.hash_cardinality
+        @state.cardinality = Cardinality.filter cardinality,
+                                                @eq_filter -
+                                                @index.hash_fields,
+                                                @range_filter
+
+        # Check if we can apply the limit from the query
+        # This occurs either when we are on the first or last index lookup
+        # and the ordering of the query has already been resolved
+        order_resolved = @state.order_by.empty? && @state.graph.size == 1
+        return unless (@state.answered?(check_limit: false) ||
+                      parent.is_a?(RootPlanStep) && order_resolved) &&
+                      !@state.query.limit.nil?
+
+        # XXX Assume that everything is limited by the limit value
+        #     which should be fine if the limit is small enough
+        @limit = @state.query.limit
+        if parent.is_a?(RootPlanStep)
+          @state.cardinality = [@limit, @state.cardinality].min
+          @state.hash_cardinality = 1
+        else
+          @limit = @state.cardinality = @state.query.limit
+
+          # If this is a final lookup by ID, go with the limit
+          if @index.graph.size == 1 && indexed_by_id
+            @state.hash_cardinality = @limit
+          else
+            @state.hash_cardinality = parent.state.cardinality
+          end
+        end
+      end
+
+      # Modify the state to reflect the fields looked up by the index
+      # @return [void]
+      def update_state(parent)
+        order_prefix = range_order_prefix.to_set
+
+        # Find fields which are filtered by the index
+        @eq_filter = @index.hash_fields + (@state.eq & order_prefix)
+        if order_prefix.include?(@state.range)
+          @range_filter = @state.range
+          @state.range = nil
+        else
+          @range_filter = nil
+        end
+
+        # Remove fields resolved by this index
+        @state.fields -= @index.all_fields
+        @state.eq -= @eq_filter
+
+        indexed_by_id = resolve_order
+        strip_graph
+        update_cardinality parent, indexed_by_id
+      end
+    end
+
+    class InvalidIndex < StandardError
+    end
+  end
+end