nose 0.1.0pre

data/lib/nose/loader/mysql.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+# This is optional so other things can run under JRuby,
+# however this loader won't work so we need to use MRI
+begin
+  require 'mysql2'
+rescue LoadError
+  require 'mysql'
+end
+
+module NoSE
+  module Loader
+    # Load data from a MySQL database into a backend
+    class MysqlLoader < LoaderBase
+      def initialize(workload = nil, backend = nil)
+        @logger = Logging.logger['nose::loader::mysqlloader']
+
+        @workload = workload
+        @backend = backend
+      end
+
+      # Load a generated set of indexes with data from MySQL
+      def load(indexes, config, show_progress = false, limit = nil,
+               skip_existing = true)
+        indexes.map!(&:to_id_graph).uniq! if @backend.by_id_graph
+
+        # XXX Assuming backend is thread-safe
+        Parallel.each(indexes, in_threads: 2) do |index|
+          load_index index, config, show_progress, limit, skip_existing
+        end
+      end
+
+      # Read all tables in the database and construct a workload object
+      def workload(config)
+        client = new_client config
+
+        workload = Workload.new
+        results = if @array_options
+                    client.query('SHOW TABLES').each(**@array_options)
+                  else
+                    client.query('SHOW TABLES').each
+                  end
+
+        results.each do |table, *|
+          # TODO: Handle foreign keys
+          workload << entity_for_table(client, table)
+        end
+
+        workload
+      end
+
+      private
+
+      # Create a new client from the given configuration
+      def new_client(config)
+        if Object.const_defined?(:Mysql2)
+          @query_options = { stream: true, cache_rows: false }
+          @array_options = { as: :array }
+          Mysql2::Client.new host: config[:host],
+                             username: config[:username],
+                             password: config[:password],
+                             database: config[:database]
+        else
+          @query_options = false
+          @array_options = false
+          Mysql.connect config[:host], config[:username], config[:password],
+                        config[:database]
+        end
+      end
+
+      # Load a single index into the backend
+      # @return [void]
+      def load_index(index, config, show_progress, limit, skip_existing)
+        client = new_client config
+
+        # Skip this index if it's not empty
+        if skip_existing && !@backend.index_empty?(index)
+          @logger.info "Skipping index #{index.inspect}" if show_progress
+          return
+        end
+        @logger.info index.inspect if show_progress
+
+        sql, fields = index_sql index, limit
+        results = if @query_options
+                    client.query(sql, **@query_options)
+                  else
+                    client.query(sql).map { |row| hash_from_row row, fields }
+                  end
+
+        result_chunk = []
+        results.each do |result|
+          result_chunk.push result
+          next if result_chunk.length < 1000
+
+          @backend.index_insert_chunk index, result_chunk
+          result_chunk = []
+        end
+        @backend.index_insert_chunk index, result_chunk \
+          unless result_chunk.empty?
+      end
+
+      # Construct a hash from the given row returned by the client
+      # @return [Hash]
+      def hash_from_row(row, fields)
+        row_hash = {}
+        fields.each_with_index do |field, i|
+          value = field.class.value_from_string row[i]
+          row_hash[field.id] = value
+        end
+
+        row_hash
+      end
+
+      # Get all the fields selected by this index
+      def index_sql_select(index)
+        fields = index.hash_fields.to_a + index.order_fields + index.extra.to_a
+
+        [fields, fields.map do |field|
+          "#{field.parent.name}.#{field.name} AS " \
+          "#{field.parent.name}_#{field.name}"
+        end]
+      end
+
+      # Get the list of tables along with the join condition
+      # for a query to fetch index data
+      # @return [String]
+      def index_sql_tables(index)
+        # Create JOIN statements
+        tables = index.graph.entities.map(&:name).join ' JOIN '
+        return tables if index.graph.size == 1
+
+        tables << ' WHERE '
+        tables << index.path.each_cons(2).map do |_prev_key, key|
+          key = key.reverse if key.relationship == :many
+          "#{key.parent.name}.#{key.name}=" \
+            "#{key.entity.name}.#{key.entity.id_field.name}"
+        end.join(' AND ')
+
+        tables
+      end
+
+      # Construct a SQL statement to fetch the data to populate this index
+      # @return [String]
+      def index_sql(index, limit = nil)
+        # Get all the necessary fields
+        fields, select = index_sql_select index
+
+        # Construct the join condition
+        tables = index_sql_tables index
+
+        query = "SELECT #{select.join ', '} FROM #{tables}"
+        query += " LIMIT #{limit}" unless limit.nil?
+
+        @logger.debug query
+        [query, fields]
+      end
+
+      # Generate an entity definition from a given table
+      # @return [Entity]
+      def entity_for_table(client, table)
+        entity = Entity.new table
+        count = client.query("SELECT COUNT(*) FROM #{table}").first
+        entity.count = count.is_a?(Hash) ? count.values.first : count
+
+        describe = if @array_options
+                     client.query("DESCRIBE #{table}").each(**@array_options)
+                   else
+                     client.query("DESCRIBE #{table}").each
+                   end
+
+        describe.each do |name, type, _, key|
+          field_class = key == 'PRI' ? Fields::IDField : field_class(type)
+          entity << field_class.new(name)
+        end
+
+        entity
+      end
+
+      # Produce the Ruby class used to represent a MySQL type
+      # @return [Class]
+      def field_class(type)
+        case type
+        when /datetime/
+          Fields::DateField
+        when /float/
+          Fields::FloatField
+        when /text/
+          # TODO: Get length
+          Fields::StringField
+        when /varchar\(([0-9]+)\)/
+          # TODO: Use length
+          Fields::StringField
+        when /(tiny)?int/
+          Fields::IntegerField
+        end
+      end
+    end
+  end
+end

data/lib/nose/loader/random.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module NoSE
+  module Loader
+    # Load some random data (mostly useful for testing)
+    class RandomLoader < LoaderBase
+      def initialize(workload = nil, backend = nil)
+        @logger = Logging.logger['nose::loader::randomloader']
+
+        @workload = workload
+        @backend = backend
+      end
+
+      # Load a generated set of indexes with data from MySQL
+      # @return [void]
+      def load(indexes, config, show_progress = false, limit = nil,
+               skip_existing = true)
+        limit = 1 if limit.nil?
+
+        indexes.map!(&:to_id_graph).uniq! if @backend.by_id_graph
+        indexes.uniq.each do |index|
+          load_index index, config, show_progress, limit, skip_existing
+        end
+      end
+
+      private
+
+      # Load a single index into the backend
+      # @return [void]
+      def load_index(index, _config, show_progress, limit, skip_existing)
+        # Skip this index if it's not empty
+        if skip_existing && !@backend.index_empty?(index)
+          @logger.info "Skipping index #{index.inspect}" if show_progress
+          return
+        end
+        @logger.info index.inspect if show_progress
+
+        chunk = Array.new(limit) do
+          Hash[index.all_fields.map do |field|
+            [field.id, field.random_value]
+          end]
+        end
+
+        @backend.index_insert_chunk index, chunk
+      end
+    end
+  end
+end

data/lib/nose/loader/sql.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'sequel'
+
+module NoSE
+  module Loader
+    # Load data from a MySQL database into a backend
+    class SqlLoader < LoaderBase
+      def initialize(workload = nil, backend = nil)
+        @logger = Logging.logger['nose::loader::sqlloader']
+
+        @workload = workload
+        @backend = backend
+      end
+
+      # Load a generated set of indexes with data from MySQL
+      def load(indexes, config, show_progress = false, limit = nil,
+               skip_existing = true)
+        indexes.map!(&:to_id_graph).uniq! if @backend.by_id_graph
+
+        # XXX Assuming backend is thread-safe
+        Parallel.each(indexes, in_threads: 2) do |index|
+          client = new_client config
+
+          # Skip this index if it's not empty
+          if skip_existing && !@backend.index_empty?(index)
+            @logger.info "Skipping index #{index.inspect}" if show_progress
+            next
+          end
+          @logger.info index.inspect if show_progress
+
+          query = index_sql client, index, limit
+
+          result_chunk = []
+          query.each do |result|
+            result = Hash[result.map { |k, v| [k.to_s, v] }]
+            result_chunk.push result
+            if result_chunk.length >= 100
+              @backend.index_insert_chunk index, result_chunk
+              result_chunk = []
+            end
+          end
+          @backend.index_insert_chunk index, result_chunk \
+            unless result_chunk.empty?
+        end
+      end
+
+      private
+
+      # Create a new client from the given configuration
+      def new_client(config)
+        Sequel.connect config[:uri]
+      end
+
+      # Get all the fields selected by this index
+      # @return [Array<String>]
+      def index_sql_select(index)
+        fields = index.hash_fields.to_a + index.order_fields + index.extra.to_a
+
+        fields.map do |field|
+          "#{field.parent.name}__#{field.name}___" \
+            "#{field.parent.name}_#{field.name}".to_sym
+        end
+      end
+
+      # Get the list of tables along with the join condition
+      # for a query to fetch index data
+      def index_sql_tables(index)
+        # Create JOIN statements
+        tables = index.graph.entities.map { |entity| entity.name.to_sym }
+        return [tables, []] if index.graph.size == 1
+
+        keys = index.path.each_cons(2).map do |_prev_key, key|
+          is_many = key.relationship == :many
+          key = key.reverse if is_many
+          fields = [key.entity.id_field.name.to_sym, key.name.to_sym]
+          fields = fields.reverse if is_many
+          Hash[[fields]]
+        end
+
+        [tables, keys]
+      end
+
+      # Construct a SQL statement to fetch the data to populate this index
+      def index_sql(client, index, limit = nil)
+        # Get all the necessary fields
+        select = index_sql_select index
+
+        # Construct the join condition
+        tables, keys = index_sql_tables index
+
+        query = client[tables.first]
+        keys.map.with_index do |key, i|
+          query = query.join tables[i + 1], key
+        end
+
+        query = query.select(*select)
+        query = query.limit limit unless limit.nil?
+
+        @logger.debug { query.sql }
+        query
+      end
+    end
+  end
+end

data/lib/nose/loader.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module NoSE
+  # Loaders which insert data into indexes from external sources
+  module Loader
+    # Superclass for all data loaders
+    class LoaderBase
+      def initialize(workload = nil, backend = nil)
+        @workload = workload
+        @backend = backend
+      end
+
+      # :nocov:
+      # @abstract Subclasses should produce a workload
+      # @return [void]
+      def workload(_config)
+        fail NotImplementedError
+      end
+      # :nocov:
+
+      # :nocov:
+      # @abstract Subclasses should load data for the given list of indexes
+      # @return [void]
+      def load(_indexes, _config, _show_progress = false, _limit = nil,
+               _skip_existing = true)
+        fail NotImplementedError
+      end
+      # :nocov:
+
+      # @abstract Subclasses should generate a model from the external source
+      # :nocov:
+      def model(_config)
+        fail NotImplementedError
+      end
+      # :nocov:
+    end
+  end
+end

data/lib/nose/model/entity.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module NoSE
+  # A representation of an object in the conceptual data model
+  class Entity
+    attr_reader :fields
+    attr_reader :foreign_keys, :name
+    attr_accessor :count
+
+    def initialize(name, &block)
+      @name = name
+      @fields = {}
+      @foreign_keys = {}
+      @count = 1
+
+      # Precompute the hash
+      hash
+
+      # Apply the DSL
+      EntityDSL.new(self).instance_eval(&block) if block_given?
+    end
+
+    # :nocov:
+    # @return [String]
+    def to_color
+      "[light_blue]#{@name}[/] [#{fields.each_key.map(&:to_color).join ', '}]"
+    end
+    # :nocov:
+
+    # Compare by name
+    # @return [Boolean]
+    def ==(other)
+      @name == other.instance_variable_get(:@name)
+    end
+    alias eql? ==
+
+    # The hash is based on the name of the entity and its fields
+    # @return [Fixnum]
+    def hash
+      @hash ||= @name.hash
+    end
+
+    # Get the key fields for the entity
+    # @return [Fields::IDField>]
+    def id_field
+      fields.each_value.find(&:primary_key?)
+    end
+
+    # Adds a {Fields::Field} to the entity
+    # @return [self] the current entity to allow chaining
+    def <<(field, freeze: true)
+      if field.is_a? Fields::ForeignKeyField
+        @foreign_keys[field.name] = field
+      else
+        @fields[field.name] = field
+      end
+
+      field.instance_variable_set(:@parent, self)
+      field.hash
+      field.freeze if freeze
+
+      self
+    end
+
+    # Shortcut for {#count=}
+    # @return [Entity]
+    def *(other)
+      fail TypeError, 'count must be an integer' unless other.is_a? Integer
+      @count = other
+
+      self
+    end
+
+    # Get the field on the entity with the given name
+    # @return [Field]
+    def [](field_name)
+      field = @fields[field_name] || @foreign_keys[field_name]
+      fail FieldNotFound if field.nil?
+      field
+    end
+
+    # Return true if the entity contains a field with the given name
+    def field?(field)
+      @fields.key? field
+    end
+
+    # Generate a hash with random values for fields in the entity
+    # @return [Hash]
+    def random_entity(prefix_entity = true)
+      Hash[@fields.map do |name, field|
+        key = name
+        key = "#{@name}_#{name}" if prefix_entity
+        [key, field.random_value]
+      end]
+    end
+  end
+
+  # A helper class for DSL creation to avoid messing with {Entity}
+  class EntityDSL
+    def initialize(entity)
+      @entity = entity
+    end
+
+    # rubocop:disable MethodName
+
+    # Specify a list of field names for the primary key
+    def PrimaryKey(*names)
+      # Unset the old keys and set new ones,
+      # we dup because the fields are frozen
+      @entity.fields.each_value do |field|
+        next unless field.primary_key?
+        field = field.dup
+        field.primary_key = false
+        @entity.fields[field.name] = field
+        field.freeze
+      end
+
+      names.each do |name|
+        field = @entity[name].dup
+        field.primary_key = true
+        @entity.fields[name] = field
+        field.freeze
+      end
+    end
+
+    # rubocop:enable MethodName
+
+    def etc(size = 1)
+      @entity << Fields::HashField.new('**', size)
+    end
+  end
+
+  # Raised when looking up a field on an entity which does not exist
+  class FieldNotFound < StandardError
+  end
+end