nose 0.1.0pre

data/lib/nose/plans/update_planner.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+module NoSE
+  module Plans
+    # A simple state class to hold the cardinality for updates
+    class UpdateState
+      attr_reader :statement, :cardinality
+
+      def initialize(statement, cardinality)
+        @statement = statement
+        @cardinality = cardinality
+      end
+
+      # XXX This is just a placeholder since this is
+      #     currently used by the query planner
+      # @return [Boolean]
+      def answered?
+        true
+      end
+    end
+
+    # A plan for executing an update
+    class UpdatePlan < AbstractPlan
+      attr_reader :statement, :index, :query_plans, :update_steps, :cost_model,
+                  :update_fields
+
+      include Comparable
+
+      def initialize(statement, index, trees, update_steps, cost_model)
+        @statement = statement
+        @index = index
+        @trees = trees
+        @query_plans = nil # these will be set later when we pick indexes
+        update_steps.each { |step| step.calculate_cost cost_model }
+        @update_steps = update_steps
+        @cost_model = cost_model
+
+        # Update with fields specified in the settings and conditions
+        # (rewrite from foreign keys to IDs if needed)
+        @update_fields = if statement.is_a?(Connection) ||
+                            statement.is_a?(Delete)
+                           []
+                         else
+                           statement.settings.map(&:field)
+                         end
+        @update_fields += statement.conditions.each_value.map(&:field)
+        @update_fields.map! do |field|
+          field.is_a?(Fields::ForeignKeyField) ? field.entity.id_field : field
+        end
+      end
+
+      # The weight of this query for a given workload
+      # @return [Fixnum]
+      def weight
+        return 1 if @workload.nil?
+
+        @workload.statement_weights[@statement]
+      end
+
+      # The group of the associated statement
+      # @return [String]
+      def group
+        @statement.group
+      end
+
+      # Name the plan by the statement
+      # @return [String]
+      def name
+        "#{@statement.text} for #{@index.key}"
+      end
+
+      # The steps for this plan are the update steps
+      # @return [Array<UpdatePlanStep>]
+      def steps
+        @update_steps
+      end
+
+      # Parameters to this update plan
+      # @return [Hash]
+      def params
+        conditions = if @statement.respond_to?(:conditions)
+                       @statement.conditions
+                     else
+                       {}
+                     end
+        settings = if @statement.respond_to?(:settings)
+                     @statement.settings
+                   else
+                     []
+                   end
+
+        params = conditions.merge Hash[settings.map do |setting|
+          [setting.field.id, Condition.new(setting.field, :'=', setting.value)]
+        end]
+
+        convert_param_keys params
+      end
+
+      # Select query plans to actually use here
+      # @return [void]
+      def select_query_plans(indexes = nil, &block)
+        if block_given?
+          @query_plans = @trees.map(&block)
+        else
+          @query_plans = @trees.map do |tree|
+            plan = tree.select_using_indexes(indexes).min_by(&:cost)
+            fail if plan.nil?
+            plan
+          end
+        end
+
+        update_support_fields
+
+        @trees = nil
+      end
+
+      # Compare all the fields for the plan for equality
+      # @return [Boolean]
+      def eql?(other)
+        return false unless other.is_a? UpdatePlan
+        fail 'plans must be resolved before checking equality' \
+          if @query_plans.nil? || other.query_plans.nil?
+
+        @statement == other.statement &&
+          @index == other.index &&
+          @query_plans == other.query_plans &&
+          @update_steps == other.update_steps &&
+          @cost_model == other.cost_model
+      end
+
+      # :nocov:
+      def to_color
+        "\n   statement: " + @statement.to_color +
+          "\n       index: " + @index.to_color +
+          "\n query_plans: " + @query_plans.to_color +
+          "\nupdate_steps: " + @update_steps.to_color +
+          "\n  cost_model: " + @cost_model.to_color
+      end
+      # :nocov:
+
+      # Two plans are compared by their execution cost
+      # @return [Boolean]
+      def <=>(other)
+        cost <=> other.cost
+      end
+
+      # The cost of performing the update on this index
+      # @return [Fixnum]
+      def update_cost
+        @update_steps.sum_by(&:cost)
+      end
+
+      # The cost is the sum of all the query costs plus the update costs
+      # @return [Fixnum]
+      def cost
+        @query_plans.sum_by(&:cost) + update_cost
+      end
+
+      private
+
+      # Add fields from support queries to those which should be updated
+      # @return [void]
+      def update_support_fields
+        # Add fields fetched from support queries
+        @update_fields += @query_plans.flat_map do |query_plan|
+          query_plan.query.select.to_a
+        end.compact
+      end
+
+      # Ensure we only use primary keys for conditions
+      # @return [Hash]
+      def convert_param_keys(params)
+        Hash[params.each_value.map do |condition|
+          field = condition.field
+          if field.is_a?(Fields::ForeignKeyField)
+            field = field.entity.id_field
+            condition = Condition.new field, condition.operator,
+                                      condition.value
+          end
+
+          [field.id, condition]
+        end]
+      end
+    end
+
+    # A planner for update statements in the workload
+    class UpdatePlanner
+      def initialize(model, trees, cost_model, by_id_graph = false)
+        @logger = Logging.logger['nose::update_planner']
+
+        @model = model
+        @cost_model = cost_model
+        @by_id_graph = by_id_graph
+
+        # Remove anything not a support query then group by statement and index
+        @query_plans = trees.select do |tree|
+          tree.query.is_a? SupportQuery
+        end
+        @query_plans = @query_plans.group_by { |tree| tree.query.statement }
+        @query_plans.each do |plan_stmt, plan_trees|
+          @query_plans[plan_stmt] = plan_trees.group_by do |tree|
+            index = tree.query.index
+            index = index.to_id_path if @by_id_path
+
+            index
+          end
+        end
+      end
+
+      # Find the necessary update plans for a given set of indexes
+      # @return [Array<UpdatePlan>]
+      def find_plans_for_update(statement, indexes)
+        indexes = indexes.map(&:to_id_graph).to_set if @by_id_graph
+
+        indexes.map do |index|
+          next unless statement.modifies_index?(index)
+
+          if (@query_plans[statement] &&
+              @query_plans[statement][index]).nil?
+            trees = []
+
+            if statement.is_a? Insert
+              cardinality = 1
+            else
+              cardinality = Cardinality.filter index.entries,
+                                               statement.eq_fields,
+                                               statement.range_field
+            end
+          else
+            # Get the cardinality of the last step to use for the update state
+            trees = @query_plans[statement][index]
+            plans = trees.map do |tree|
+              tree.select_using_indexes(indexes).min_by(&:cost)
+            end
+
+            # Multiply the cardinalities because we are crossing multiple
+            # relationships and need the cross-product
+            cardinality = plans.product_by { |p| p.last.state.cardinality }
+          end
+
+          state = UpdateState.new statement, cardinality
+          update_steps = update_steps statement, index, state
+          UpdatePlan.new statement, index, trees, update_steps, @cost_model
+        end.compact
+      end
+
+      private
+
+      # Find the required update steps
+      # @return [Array<UpdatePlanStep>]
+      def update_steps(statement, index, state)
+        update_steps = []
+        update_steps << DeletePlanStep.new(index, state) \
+          if statement.requires_delete?(index)
+
+        if statement.requires_insert?(index)
+          fields = if statement.is_a?(Connect)
+                     statement.conditions.each_value.map(&:field)
+                   else
+                     statement.settings.map(&:field)
+                   end
+
+          update_steps << InsertPlanStep.new(index, state, fields)
+        end
+
+        update_steps
+      end
+    end
+  end
+end

data/lib/nose/plans.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module NoSE
+  # Statement planning and abstract models of execution steps
+  module Plans
+    # A single step in a statement plan
+    class PlanStep
+      include Supertype
+
+      attr_accessor :state, :parent
+      attr_reader :children, :cost, :fields
+
+      def initialize
+        @children = Set.new
+        @parent = nil
+        @fields = Set.new
+      end
+
+      # :nocov:
+      def to_color
+        # Split on capital letters and remove the last two parts (PlanStep)
+        self.class.name.split('::').last.split(/(?=[A-Z])/)[0..-3] \
+          .map(&:downcase).join(' ').capitalize
+      end
+      # :nocov:
+
+      # Set the children of the current plan step
+      # @return [void]
+      def children=(children)
+        @children = children.to_set
+
+        # Track the parent step of each step
+        children.each do |child|
+          child.instance_variable_set(:@parent, self)
+          fields = child.instance_variable_get(:@fields) + self.fields
+          child.instance_variable_set(:@fields, fields)
+        end
+      end
+
+      # Mark the fields in this index as fetched
+      # @return [void]
+      def add_fields_from_index(index)
+        @fields += index.all_fields
+      end
+
+      # Get the list of steps which led us here
+      # If a cost model is not provided, statement plans using
+      # this step cannot be evaluated on the basis of cost
+      #
+      # (this is to support PlanStep#parent_index which does not need cost)
+      # @return [QueryPlan]
+      def parent_steps(cost_model = nil)
+        steps = nil
+
+        if @parent.nil?
+          steps = QueryPlan.new state.query, cost_model
+        else
+          steps = @parent.parent_steps cost_model
+          steps << self
+        end
+
+        steps
+      end
+
+      # Find the closest index to this step
+      # @return [PlanStep]
+      def parent_index
+        step = parent_steps.to_a.reverse_each.find do |parent_step|
+          parent_step.is_a? IndexLookupPlanStep
+        end
+        step.index unless step.nil?
+      end
+
+      # Calculate the cost of executing this step in the plan
+      # @return [Fixnum]
+      def calculate_cost(cost_model)
+        @cost = cost_model.method((subtype_name + '_cost').to_sym).call self
+      end
+
+      # Add the Subtype module to all step classes
+      # @return [void]
+      def self.inherited(child_class)
+        child_class.send(:include, Subtype)
+      end
+    end
+
+    # A dummy step used to inspect failed statement plans
+    class PrunedPlanStep < PlanStep
+      def state
+        OpenStruct.new answered?: true
+      end
+    end
+
+    # The root of a tree of statement plans used as a placeholder
+    class RootPlanStep < PlanStep
+      def initialize(state)
+        super()
+        @state = state
+        @cost = 0
+      end
+    end
+
+    # This superclass defines what is necessary for manually defined
+    # and automatically generated plans to provide for execution
+    class AbstractPlan
+      attr_reader :group, :name, :weight
+
+      # @abstract Subclasses should produce the steps for executing this query
+      def steps
+        fail NotImplementedError
+      end
+
+      # @abstract Subclasses should produce the fields selected by this plan
+      def select_fields
+        []
+      end
+
+      # @abstract Subclasses should produce the parameters
+      # necessary for this plan
+      def params
+        fail NotImplementedError
+      end
+    end
+  end
+end
+
+require_relative 'plans/filter'
+require_relative 'plans/index_lookup'
+require_relative 'plans/limit'
+require_relative 'plans/sort'
+require_relative 'plans/update'
+
+require_relative 'plans/query_planner'
+require_relative 'plans/update_planner'
+require_relative 'plans/execution_plan'

data/lib/nose/proxy/mysql.rb

@@ -0,0 +1,275 @@
+# frozen_string_literal: true
+
+require 'mysql'
+
+module NoSE
+  module Proxy
+    # A proxy which speaks the MySQL protocol and executes queries
+    class MysqlProxy < ProxyBase
+      def initialize(*args)
+        super
+
+        # Initialize a hash for the state of sockets
+        @state = {}
+      end
+
+      # Authenticate the client and process queries
+      def handle_connection(socket)
+        return authenticate socket if @state[socket].nil?
+
+        # Retrieve the saved state of the socket
+        protocol = @state[socket]
+
+        begin
+          protocol.process_command(&method(:process_query))
+        rescue ::Mysql::ClientError::ServerGoneError
+          # Ensure the socket is closed and remove the state
+          remove_connection socket
+          return false
+        end
+
+        # Keep this socket around
+        true
+      end
+
+      # Remove the state of the socket
+      def remove_connection(socket)
+        socket.close
+        @state.delete socket
+      end
+
+      private
+
+      # Auth the client and prepare for query processsing
+      # @return [Boolean]
+      def authenticate(socket)
+        protocol = ::Mysql::ServerProtocol.new socket
+
+        # Try to authenticate
+        begin
+          protocol.authenticate
+        rescue
+          remove_connection socket
+          return false
+        end
+
+        @state[socket] = protocol
+
+        true
+      end
+
+      # Execute the query on the backend and return the result
+      def process_query(protocol, query)
+        begin
+          @logger.debug { "Got query #{query}" }
+          result = query_result query
+          @logger.debug "Executed query with #{result.size} results"
+        rescue ParseFailed => exc
+          protocol.error ::Mysql::ServerError::ER_PARSE_ERROR, exc.message
+        rescue Backend::PlanNotFound => exc
+          protocol.error ::Mysql::ServerError::ER_UNKNOWN_STMT_HANDLER,
+                         exc.message
+        end
+
+        result
+      end
+
+      private
+
+      # Get the result of the query from the backend
+      def query_result(query)
+        query = Statement.parse query, @result.workload.model
+        @backend.query(query).lazy.map do |row|
+          Hash[query.select.map { |field| [field.name, row[field.id]] }]
+        end
+      end
+    end
+  end
+
+  # Extend the client library with necessary server code
+  class ::Mysql
+    # Simple class which doesn't do connection setup
+    class ServerProtocol < Protocol
+      def initialize(socket)
+        # We need a much simpler initialization than the default class
+        @sock = socket
+      end
+
+      # Perform authentication
+      def authenticate
+        reset
+        write InitialPacket.serialize
+        AuthenticationPacket.parse read # TODO: Check auth
+        write ResultPacket.serialize 0
+      end
+
+      # Send an error message with the given number and text
+      def error(errno, message)
+        write ErrorPacket.serialize errno, message
+      end
+
+      # Process a single incoming command
+      def process_command(&block)
+        reset
+        pkt = read
+        command = pkt.utiny
+
+        case command
+        when COM_QUIT
+          # Stop processing because the client left
+          return
+        when COM_QUERY
+          process_query pkt.to_s, &block
+        when COM_PING
+          write ResultPacket.serialize 0
+        else
+          # Return error for invalid commands
+          protocol.error ::Mysql::ServerError::ER_NOT_SUPPORTED_YET,
+                         'Command not supported'
+        end
+      end
+
+      private
+
+      # Handle an individual query
+      def process_query(query)
+        # Execute the query on the backend
+        result = yield self, query
+        return if result.nil?
+
+        # Return the list of fields in the result
+        field_names = result.any? ? result.peek.keys : []
+        write_fields result, field_names
+        write_rows result, field_names
+      end
+
+      # Write the list of fields for the resulting rows
+      def write_fields(result, field_names)
+        write ResultPacket.serialize field_names.count
+        field_names.each do |field_name|
+          type, = Protocol.value2net result.first[field_name]
+
+          write FieldPacket.serialize '', '', '', field_name, '', 1, type,
+                                      Field::NOT_NULL_FLAG, 0, ''
+        end
+        write EOFPacket.serialize
+      end
+
+      # Write a packet for each row in the results
+      def write_rows(result, field_names)
+        result.each do |row|
+          values = field_names.map { |field_name| row[field_name] }
+          write(values.map do |value|
+            Protocol.value2net(value.to_s).last
+          end.inject('', &:+))
+        end
+        write EOFPacket.serialize
+      end
+    end
+
+    # Add serialization of the initial packet
+    class InitialPacket
+      # Serialize the initial server hello
+      # @return [String]
+      def self.serialize
+        [
+          ::Mysql::Protocol::VERSION,
+          'nose',
+          0,
+          'AAAAAAAA',
+          0,
+          CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION,
+          33, # utf8_general_ci
+          SERVER_STATUS_AUTOCOMMIT,
+          'AAAAAAAAAAAA'
+        ].pack('CZ*Va8CvCvx13Z*')
+      end
+    end
+
+    # Add serialization of result packets
+    class ResultPacket
+      # Serialize a simple OK response
+      # rubocop:disable Metrics/ParameterLists
+      # @return [String]
+      def self.serialize(field_count, affected_rows = 0, insert_id = 0,
+                         server_status = 0, warning_count = 0, message = '')
+        return Packet.lcb(field_count) unless field_count.zero?
+
+        Packet.lcb(field_count) +
+          Packet.lcb(affected_rows) +
+          Packet.lcb(insert_id) +
+          [
+            server_status,
+            warning_count
+          ].pack('vv') +
+          Packet.lcs(message)
+      end
+      # rubocop:enable Metrics/ParameterLists
+    end
+
+    # Add serialization of field packets
+    class FieldPacket
+      # Serialize all the data for a field
+      # rubocop:disable Metrics/ParameterLists
+      # @return [String]
+      def self.serialize(db, table, org_table, name, org_name, length, type,
+                         flags, decimals, default)
+        Packet.lcs('def') + # catalog
+          Packet.lcs(db) +
+          Packet.lcs(table) +
+          Packet.lcs(org_table) +
+          Packet.lcs(name) +
+          Packet.lcs(org_name) +
+          [
+            0x0c,
+            33, # utf8_general_ci
+            length,
+            type,
+            flags,
+            decimals,
+            0
+          ].pack('CvVCvCv') + Packet.lcs(default)
+      end
+      # rubocop:enable Metrics/ParameterLists
+    end
+
+    # Add parsing of auth packets
+    class AuthenticationPacket
+      # Parse the incoming authentication packet
+      def self.parse(_pkt)
+        # XXX: Unneeded for now since we don't handle auth
+        # client_flags = pkt.ulong
+        # max_packet_size = pkt.ulong
+        # charset_number = pkt.lcb
+        # f1 = pkt.read(23)
+        # username = pkt.string
+        # scrambled_password = pkt.lcs
+        # databasename = pkt.string
+      end
+    end
+
+    # Simple EOF packet
+    class EOFPacket
+      # Static string to indicate EOF
+      # @return [String]
+      def self.serialize
+        "\xfe\x00\x00\x00\x00"
+      end
+    end
+
+    # Serialize an error message
+    class ErrorPacket
+      # Generate a packet with a given error number and message
+      # @return [String]
+      def self.serialize(errno, message)
+        [
+          0xff,
+          errno,
+          '#',
+          @sqlstate,
+          message
+        ].pack('Cvaa5a*')
+      end
+    end
+  end
+end