nose 0.1.0pre

data/lib/nose/util.rb

@@ -0,0 +1,305 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'formatador'
+require 'parallel'
+require 'pp'
+require 'stringio'
+
+# Reopen to add utility methods
+module Enumerable
+  # Enumerate all non-empty prefixes of the enumerable
+  # @return [Enumerator]
+  def prefixes
+    Enumerator.new do |enum|
+      prefix = []
+      each do |elem|
+        prefix = prefix.dup << elem
+        enum.yield prefix
+      end
+    end
+  end
+
+  # Enumerate all partitionings of an enumerable
+  # @return [Enumerator]
+  def partitions(max_length = nil)
+    max_length = length if max_length.nil?
+    Enumerator.new do |enum|
+      1.upto(max_length).map do |length|
+        enum.yield partition.with_index { |_, i| i < length }
+      end
+    end
+  end
+
+  # Take the sum of the result of calling the block on each item
+  # @return [Object]
+  def sum_by(initial = 0)
+    reduce(initial) { |sum, item| sum + yield(item) }
+  end
+
+  # Take the product of the result of calling the block on each item
+  # @return [Object]
+  def product_by(initial = 1)
+    reduce(initial) { |product, item| product * yield(item) }
+  end
+end
+
+# Extend with some convenience methods
+class Array
+  # Find the longest common prefix of two arrays
+  # @return [Array<Object>]
+  def longest_common_prefix(other)
+    fail TypeError unless other.is_a? Array
+    (prefixes.to_a & other.prefixes.to_a).max_by(&:length) || []
+  end
+end
+
+# Reopen to present as finite as with Float
+class Integer
+  # Convenience methods to allow integers to be considered finite
+  # @return [Boolean]
+  def finite?
+    true
+  end
+end
+
+# Extend Object to print coloured output
+class Object
+  def inspect
+    Formatador.parse(respond_to?(:to_color) ? to_color : to_s)
+  end
+
+  # Get a colored representation of the object
+  # @return [String]
+  def to_color
+    to_s
+  end
+end
+
+# Allow a supertype to look up a class given the
+# name of a subtype inheriting from this class
+module Supertype
+  # Add class methods when this module is included
+  # @return [void]
+  def self.included(base)
+    base.extend ClassMethods
+  end
+
+  # Add a single method to get a class given the subtype name
+  module ClassMethods
+    # Get the class given the name of a subtype
+    # @return [Class] the concrete class with the given subtype name
+    def subtype_class(name)
+      class_name = self.name.split('::')[0..-2]
+      class_name << name.split('_').map do |name_part|
+        name_part = name_part[0].upcase + name_part[1..-1]
+        name_part.sub 'Id', 'ID'
+      end.join
+      class_name[-1] = class_name[-1] + self.name.split('::').last
+
+      class_name.reduce(Object) do |mod, name_part|
+        mod.const_get(name_part)
+      end
+    end
+  end
+end
+
+# Allow subclasses to return a string representing of the
+# class, minus a common suffix also used in the superclass
+module Subtype
+  # Add instance and class methods when this module is included
+  # @return [void]
+  def self.included(base)
+    base.send :include, InstanceMethods
+    base.extend ClassMethods
+  end
+
+  # Mirror the subtype method on class instances
+  module InstanceMethods
+    # A mirror of {Subtype::ClassMethods#subtype_name}
+    # @return [String]
+    def subtype_name(**args)
+      self.class.subtype_name(**args)
+    end
+  end
+
+  # Add a single method to retrieve the subtype name
+  module ClassMethods
+    # Get a unique string identify this subclass amongst sibling classes
+    # @return [String]
+    def subtype_name(name_case: :snake)
+      super_name = name_array superclass
+      self_name = name_array self
+      self_name = self_name.reverse.drop_while do |part|
+        super_name.include? part
+      end.reverse
+
+      if name_case == :snake
+        name = self_name.join('_').freeze
+      elsif name_case == :camel
+        name = self_name.map do |part|
+          part[0].upcase + part[1..-1]
+        end.join('').freeze
+      end
+
+      name
+    end
+
+    private
+
+    # Convert camel case class names to an array
+    # @return [Array<String>]
+    def name_array(cls)
+      frozen_name = cls.name
+      frozen_name.gsub!(/^.*::/, '')
+      frozen_name.gsub!('ID', 'Id')
+      frozen_name.freeze
+
+      frozen_name.split(/(?=[A-Z]+)/).map(&:freeze) \
+                 .map! do |s|
+        s.downcase.freeze
+      end
+    end
+  end
+end
+
+# Simple helper class to facilitate cardinality estimates
+class Cardinality
+  # Update the cardinality based on filtering implicit to the index
+  # @return [Fixnum]
+  def self.filter(cardinality, eq_filter, range_filter)
+    filtered = (range_filter.nil? ? 1.0 : 0.1) * cardinality
+    filtered *= eq_filter.map do |field|
+      1.0 / field.cardinality
+    end.inject(1.0, &:*)
+
+    filtered
+  end
+end
+
+# Add a simple function for pretty printing strings
+module Kernel
+  private
+
+  # Pretty print to a string
+  # @return [String]
+  def pp_s(*objs)
+    s = StringIO.new
+    objs.each { |obj| PP.pp(obj, s) }
+    s.rewind
+    s.read
+  end
+
+  module_function :pp_s
+end
+
+# Add simple convenience methods
+class Object
+  # Convert all the keys of a hash to symbols
+  # @return [Object]
+  def deep_symbolize_keys
+    return each_with_object({}) do |(k, v), memo|
+      memo[k.to_sym] = v.deep_symbolize_keys
+      memo
+    end if is_a? Hash
+
+    return each_with_object([]) do |v, memo|
+      memo << v.deep_symbolize_keys
+      memo
+    end if is_a? Array
+
+    self
+  end
+end
+
+# Extend the kernel to allow warning suppression
+module Kernel
+  # Allow the suppression of warnings for a block of code
+  # @return [void]
+  def suppress_warnings
+    original_verbosity = $VERBOSE
+    $VERBOSE = nil
+    result = yield
+    $VERBOSE = original_verbosity
+
+    result
+  end
+end
+
+module NoSE
+  # Helper functions for building DSLs
+  module DSL
+    # Add methods to the class which can be used to access entities and fields
+    # @return [void]
+    def mixin_fields(entities, cls)
+      entities.each do |entity_name, entity|
+        # Add fake entity object for the DSL
+        fake = Object.new
+
+        # Add a method named by the entity to allow field creation
+        cls.send :define_method, entity_name.to_sym, (proc do
+          metaclass = class << fake; self; end
+
+          # Allow fields to be defined using [] access
+          metaclass.send :define_method, :[] do |field_name|
+            if field_name == '*'
+              entity.fields.values
+            else
+              entity.fields[field_name] || entity.foreign_keys[field_name]
+            end
+          end
+
+          # Define methods named for fields so things like 'user.id' work
+          entity.fields.merge(entity.foreign_keys).each do |field_name, field|
+            metaclass.send :define_method, field_name.to_sym, -> { field }
+          end
+
+          fake
+        end)
+      end
+    end
+
+    module_function :mixin_fields
+  end
+
+  # Add loading of class instances from the filesystem
+  module Loader
+    attr_reader :source_code
+
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    # Add a class method to load class instances from file
+    module ClassMethods
+      # Load a class with the given name from a directory specified
+      # by the LOAD_PATH class constant
+      # @return [Object] an instance of the class which included this module
+      def load(name)
+        path = const_get(:LOAD_PATH)
+        filename = File.expand_path "../../../#{path}/#{name}.rb", __FILE__
+        source_code = File.read(filename)
+
+        instance = binding.eval source_code, filename
+        instance.instance_variable_set :@source_code, source_code
+        instance
+      end
+    end
+  end
+end
+
+# Extend Time to allow conversion to DateTime instances
+class Time
+  # Convert to a DateTime instance
+  # http://stackoverflow.com/a/279785/123695
+  # @return [DateTime]
+  def to_datetime
+    # Convert seconds + microseconds into a fractional number of seconds
+    seconds = sec + Rational(usec, 10**6)
+
+    # Convert a UTC offset measured in minutes to one measured in a
+    # fraction of a day.
+    offset = Rational(utc_offset, 60 * 60 * 24)
+    DateTime.new(year, month, day, hour, min, seconds, offset)
+  end
+end

data/lib/nose/workload.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+require_relative 'model'
+require_relative 'parser'
+
+require 'erb'
+
+module NoSE
+  # A representation of a query workload over a given set of entities
+  class Workload
+    # The subdirectory workloads are loaded from
+    LOAD_PATH = 'workloads'
+    include Loader
+
+    attr_reader :model
+    attr_accessor :mix
+
+    def initialize(model = nil, &block)
+      @statement_weights = { default: {} }
+      @model = model || Model.new
+      @mix = :default
+
+      # Apply the DSL
+      WorkloadDSL.new(self).instance_eval(&block) if block_given?
+    end
+
+    # Compare models and statements
+    # @return [Boolean]
+    def ==(other)
+      other.is_a?(Workload) && @model == other.model &&
+        statement_weights == other.statement_weights
+    end
+    alias eql? ==
+
+    # Add a new {Entity} or {Statement} to the workload
+    # @return [self] the current workload to allow chaining
+    def <<(other)
+      if other.is_a? Entity
+        @model.add_entity other.freeze
+      elsif other.is_a? Statement
+        add_statement other.freeze
+      else
+        fail TypeError, 'can only add queries and entities to a workload'
+      end
+
+      self
+    end
+
+    # Add a new {Statement} to the workload or parse a string
+    # @return [void]
+    def add_statement(statement, mixes = {}, group: nil, label: nil)
+      statement = Statement.parse(statement, @model,
+                                  group: group, label: label) \
+        if statement.is_a? String
+      statement.freeze
+
+      mixes = { default: mixes } if mixes.is_a? Numeric
+      mixes = { default: 1.0 } if mixes.empty?
+      mixes.each do |mix, weight|
+        @statement_weights[mix] = {} unless @statement_weights.key? mix
+        @statement_weights[mix][statement] = weight
+      end
+    end
+
+    # Strip the weights from the query dictionary and return a list of queries
+    # @return [Array<Statement>]
+    def queries
+      @statement_weights[@mix].keys.select do |statement|
+        statement.is_a? Query
+      end
+    end
+
+    # Strip the weights and return a list of statements
+    # @return [Array<Statement>]
+    def statements
+      (@statement_weights[@mix] || {}).keys
+    end
+
+    # Retrieve the weights for the current mix
+    # @return [Hash]
+    def statement_weights
+      @statement_weights[@mix]
+    end
+
+    # Strip the weights from the query dictionary and return a list of updates
+    # @return [Array<Statement>]
+    def updates
+      @statement_weights[@mix].keys.reject do |statement|
+        statement.is_a? Query
+      end
+    end
+
+    # Find a statement in the workload with the provided tag
+    # @return [Statement]
+    def find_with_tag(tag)
+      statements.find do |s|
+        s.text.end_with? "-- #{tag}"
+      end
+    end
+
+    # Remove any updates from the workload
+    # @return [void]
+    def remove_updates
+      @statement_weights[@mix].select! { |stmt, _| stmt.is_a? Query }
+    end
+
+    # Get all the support queries for updates in the workload
+    # @return[Array<Statement>]
+    def support_queries(indexes)
+      updates.map do |update|
+        indexes.map { |index| update.support_queries(index) }
+      end.flatten(2)
+    end
+
+    # Check if all the fields used by queries in the workload exist
+    # @return [Boolean]
+    def fields_exist?
+      @statement_weights[@mix].each_key do |query|
+        # Projected fields and fields in the where clause exist
+        fields = query.where.map(&:field) + query.fields
+        fields.each do |field|
+          return false unless @model.find_field field.value
+        end
+      end
+
+      true
+    end
+
+    # Produce the source code used to define this workload
+    # @return [String]
+    def source_code
+      return @source_code unless @source_code.nil?
+
+      ns = OpenStruct.new(workload: self)
+      tmpl = File.read File.join(File.dirname(__FILE__),
+                                 '../../templates/workload.erb')
+      tmpl = ERB.new(tmpl, nil, '>')
+      @source_code = tmpl.result(ns.instance_eval { binding })
+    end
+  end
+
+  # A helper class for DSL creation to avoid messing with {Workload}
+  class WorkloadDSL
+    def initialize(arg)
+      if arg.is_a? Workload
+        @workload = arg
+        @model = arg.model
+      elsif arg.is_a? Model
+        @model = arg
+      end
+    end
+
+    # rubocop:disable MethodName
+
+    # Allow the use of an external model
+    def Model(name)
+      @workload.instance_variable_set(:@model, NoSE::Model.load(name))
+    end
+
+    # Shortcut to add a new {Entity} to the workload
+    # @return [Entity]
+    def Entity(*args, &block)
+      @model.add_entity Entity.new(*args, &block)
+    end
+
+    # Add a HasMany relationship which is just the opposite of HasOne
+    # @return [void]
+    def HasMany(from_name, to_name, entities, **options)
+      HasOne to_name, from_name, Hash[[entities.first.reverse]], **options
+    end
+
+    # Separate function for foreign keys to avoid circular dependencies
+    # @return [void]
+    def HasOne(from_name, to_name, entities, **options)
+      from_entity, to_entity = entities.first
+      from_field = Fields::ForeignKeyField.new from_name,
+                                               @model[to_entity],
+                                               **options
+
+      # Add the key in the opposite direction
+      options[:count] = @model[from_entity].count
+      options[:relationship] = :many
+      to_field = Fields::ForeignKeyField.new to_name,
+                                             @model[from_entity],
+                                             **options
+
+      # Set the opposite keys and add to entities
+      to_field.reverse = from_field
+      from_field.reverse = to_field
+      @model[from_entity] << from_field
+      @model[to_entity] << to_field
+    end
+
+    # Shortcut to add a new {Statement} to the workload
+    # @return [void]
+    def Q(statement, weight = 1.0, group: nil, label: nil, **mixes)
+      fail 'Statements require a workload' if @workload.nil?
+
+      return if weight.zero? && mixes.empty?
+      mixes = { default: weight } if mixes.empty?
+      @workload.add_statement statement, mixes, group: group, label: label
+    end
+
+    # Allow setting the default workload mix
+    # @return [void]
+    def DefaultMix(mix)
+      @workload.mix = mix
+    end
+
+    # Allow grouping statements with an associated weight
+    # @return [void]
+    def Group(name, weight = 1.0, **mixes, &block)
+      fail 'Groups require a workload' if @workload.nil?
+
+      # Apply the DSL
+      dsl = GroupDSL.new
+      dsl.instance_eval(&block) if block_given?
+      dsl.statements.each do |statement|
+        Q(statement, weight, **mixes, group: name)
+      end
+    end
+
+    # rubocop:enable MethodName
+  end
+
+  # A helper class for DSL creation to allow groups of statements
+  class GroupDSL
+    attr_reader :statements
+
+    def initialize
+      @statements = []
+    end
+
+    # rubocop:disable MethodName
+
+    # Track a new statement to be added
+    # @return [void]
+    def Q(statement)
+      @statements << statement
+    end
+
+    # rubocop:enable MethodName
+  end
+end

data/lib/nose.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# Namespace module for the whole project
+module NoSE
+end
+
+require_relative 'nose/util'
+
+require_relative 'nose/backend'
+require_relative 'nose/cost'
+require_relative 'nose/debug'
+require_relative 'nose/enumerator'
+require_relative 'nose/indexes'
+require_relative 'nose/loader'
+require_relative 'nose/model'
+require_relative 'nose/parser'
+require_relative 'nose/plans'
+require_relative 'nose/proxy'
+require_relative 'nose/query_graph'
+require_relative 'nose/random'
+require_relative 'nose/schema'
+require_relative 'nose/search'
+require_relative 'nose/statements'
+require_relative 'nose/timing'
+require_relative 'nose/workload'
+
+require_relative 'nose/serialize'
+
+# :nocov:
+require 'logging'
+
+logger = Logging.logger['nose']
+logger.level = (ENV['NOSE_LOG'] || 'info').downcase.to_sym
+
+logger.add_appenders Logging.appenders.stderr
+logger = nil # rubocop:disable Lint/UselessAssignment
+# :nocov:

data/templates/workload.erb

@@ -0,0 +1,42 @@
+<% foreign_keys = [] %>
+# rubocop:disable all
+
+workload = NoSE::Workload.new do
+  # Define entities along with the size and cardinality of their fields
+  # as well as an estimated number of each entity
+  <% workload.model.entities.each_value do |entity| %>
+(Entity '<%= entity.name %>' do
+  <% entity.fields.each_value do |field| %>
+  <%
+    if field.is_a? NoSE::Fields::ForeignKeyField
+      foreign_keys << field
+      next
+    end
+  %>
+<%= field.subtype_name name_case: :camel %> '<%= field.name %>',<%=
+  case [field.class]
+  when [NoSE::Fields::StringField]
+    "#{field.size}, "
+  else
+    ''
+  end
+  %> count: <%= field.cardinality %>
+
+  <% end %>
+end) * <%= entity.count %>
+
+
+  <% end %>
+
+  <% foreign_keys.each do |key| %>
+    ForeignKey '<%= key.name %>', '<%= key.parent.name %>', '<%= key.entity.name %>', count: <%= key.cardinality %>
+
+  <% end %>
+
+  # Define queries and their relative weights
+  <% workload.statement_weights.each do |statement, weight| %>
+  Q '<%= statement.query %>', <%= weight %>
+
+  <% end %>
+end
+# rubocop:enable all