low_card_tables 1.0.0

data/lib/low_card_tables/low_card_table/table_unique_index.rb

@@ -0,0 +1,134 @@
+module LowCardTables
+  module LowCardTable
+    # A TableUniqueIndex represents the concept of a unique index for a given low-card model class. I say "the concept",
+    # because there should only be one instance of this class for any given low-card model class -- there isn't one
+    # instance of this class for each actual unique index for the class in question.
+    #
+    # This class started as code that was directly part of the RowManager, and was factored out to create this class
+    # instead -- simply so that the RowManager wouldn't have any more code in it than necessary.
+    class TableUniqueIndex
+      # Creates a new instance for the low-card model class in question.
+      def initialize(low_card_model)
+        unless low_card_model.respond_to?(:is_low_card_table?) && low_card_model.is_low_card_table?
+          raise ArgumentError, "You must supply a low-card AR model class, not: #{low_card_model.inspect}"
+        end
+
+        @low_card_model = low_card_model
+      end
+
+      # Ensures that the unique index is present. If the index is present, does nothing else.
+      #
+      # If the index is not present, then looks at +create_if_needed+. If this evaluates to true, then it will create
+      # the index. If this evaluates to false, then it will raise an exception.
+      def ensure_present!(create_if_needed)
+        return unless @low_card_model.table_exists?
+
+        current_name = current_unique_all_columns_index_name
+        return true if current_name
+
+        if create_if_needed
+          create_unique_index!
+          true
+        else
+          message = %{You said that the table '#{low_card_model.table_name}' is a low-card table.
+However, it currently does not seem to have a unique index on all its columns. For the
+low-card system to work properly, this is *required* -- although the low-card system
+tries very hard to lock tables and otherwise ensure that it never will create duplicate
+rows, this is important enough that we really want the database to enforce it.
+
+We're looking for an index on the following columns:
+
+  #{value_column_names.sort.join(", ")}
+
+...and we have the following unique indexes:
+
+}
+          current_unique_indexes.each do |unique_index|
+            message << "  '#{unique_index.name}': #{unique_index.columns.sort.join(", ")}\n"
+          end
+          message << "\n"
+
+          raise LowCardTables::Errors::LowCardNoUniqueIndexError, message
+        end
+      end
+
+      # Removes the unique index, if one is present. If one is not present, does nothing.
+      def remove!
+        table_name = low_card_model.table_name
+        current_name = current_unique_all_columns_index_name
+
+        if current_name
+          migrate do
+            remove_index table_name, :name => current_name
+          end
+
+          now_current_name = current_unique_all_columns_index_name
+          if now_current_name
+            raise "Whoa -- we tried to remove the unique index on #{table_name}, which was named '#{current_name}', but, after we removed it, we still have a unique all-columns index called '#{now_current_name}'!"
+          end
+        end
+      end
+
+      private
+      attr_reader :low_card_model
+
+      def value_column_names
+        low_card_model.low_card_value_column_names
+      end
+
+      def migrate(&block)
+        migration_class = Class.new(::ActiveRecord::Migration)
+        metaclass = migration_class.class_eval { class << self; self; end }
+        metaclass.instance_eval { define_method(:up, &block) }
+
+        ::ActiveRecord::Migration.suppress_messages do
+          migration_class.migrate(:up)
+        end
+
+        low_card_model.reset_column_information
+        LowCardTables::VersionSupport.clear_schema_cache!(low_card_model)
+      end
+
+      def create_unique_index!
+        raise "Whoa -- there should never already be a unique index for #{low_card_model}!" if current_unique_all_columns_index_name
+
+        table_name = low_card_model.table_name
+        column_names = value_column_names.sort
+        ideal_name = ideal_unique_all_columns_index_name
+
+        migrate do
+          remove_index table_name, :name => ideal_name rescue nil
+          add_index table_name, column_names, :unique => true, :name => ideal_name
+        end
+
+        unless current_unique_all_columns_index_name
+          raise "Whoa -- there should always be a unique index by now for #{low_card_model}! We think we created one, but now it still doesn't exist?!?"
+        end
+      end
+
+      def current_unique_indexes
+        return [ ] if (! low_card_model.table_exists?)
+        low_card_model.connection.indexes(low_card_model.table_name).select { |i| i.unique }
+      end
+
+      def current_unique_all_columns_index_name
+        index = current_unique_indexes.detect { |index| index.columns.sort == value_column_names.sort }
+        index.name if index
+      end
+
+      # We just limit all index names to this length -- this should be the smallest maximum index-name length that
+      # any database supports.
+      MINIMUM_MAX_INDEX_NAME_LENGTH = 63
+
+      def ideal_unique_all_columns_index_name
+        index_part_1 = "index_"
+        index_part_2 = "_lc_on_all"
+
+        remaining_characters = MINIMUM_MAX_INDEX_NAME_LENGTH - (index_part_1.length + index_part_2.length)
+        index_name = index_part_1 + (@low_card_model.table_name[0..(remaining_characters - 1)]) + index_part_2
+
+        index_name
+      end
+    end
+  end
+end

data/lib/low_card_tables/version.rb

@@ -0,0 +1,4 @@
+# Defines the current version of +low_card_tables+.
+module LowCardTables
+  VERSION = "1.0.0"
+end

data/lib/low_card_tables/version_support.rb

@@ -0,0 +1,52 @@
+module LowCardTables
+  # Contains methods used by the codebase to support differing ActiveRecord versions. This is just a clean way of
+  # factoring out differing ActiveRecord API into a single class.
+  class VersionSupport
+    class << self
+      # Clear the schema cache for a given model.
+      def clear_schema_cache!(model)
+        if model.connection.respond_to?(:schema_cache)
+          model.connection.schema_cache.clear!
+        elsif model.connection.respond_to?(:clear_cache!)
+          model.connection.clear_cache!
+        end
+      end
+
+      # Can you specify a block on default_scope? This was added in ActiveRecord 3.1.
+      def default_scopes_accept_a_block?
+        ! (::ActiveRecord::VERSION::MAJOR <= 3 && ::ActiveRecord::VERSION::MINOR == 0)
+      end
+
+      # Is #migrate a class method, or an instance method, on ActiveRecord::Migration? It changed to an instance method
+      # as of ActiveRecord 3.1.
+      def migrate_is_a_class_method?
+        (::ActiveRecord::VERSION::MAJOR <= 3 && ::ActiveRecord::VERSION::MINOR == 0)
+      end
+
+      # Define a default scope on the class in question. This is only actually used from our specs.
+      def define_default_scope(klass, conditions)
+        if default_scopes_accept_a_block?
+          if conditions
+            klass.instance_eval %{
+    default_scope { where(#{conditions.inspect}) }
+}
+          else
+            klass.instance_eval %{
+    default_scope { }
+}
+          end
+        else
+          if conditions
+            klass.instance_eval %{
+    default_scope where(#{conditions.inspect})
+}
+          else
+            klass.instance_eval %{
+    default_scope nil
+}
+          end
+        end
+      end
+    end
+  end
+end

data/low_card_tables.gemspec

@@ -0,0 +1,69 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "low_card_tables/version"
+
+Gem::Specification.new do |s|
+  s.name        = "low_card_tables"
+  s.version     = LowCardTables::VERSION
+  s.authors     = ["Andrew Geweke"]
+  s.email       = ["andrew@geweke.org"]
+  s.homepage    = "https://github.com/ageweke/low_card_tables"
+  s.summary     = %q{"Bitfields for ActiveRecord": instead of storing multiple columns with low cardinality (few distinct values) directly in a table, which results in performance and maintainability problems, break them out into a separate table with almost zero overhead. Trivially add new columns without migrating a main, enormous table. Query on combinations of values very efficiently.}
+  s.description = %q{"Bitfields for ActiveRecord": store low-cardinality columns in a separate table for vastly more flexibility and better performance.}
+  s.license     = "MIT"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "bundler"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec", "~> 2.14"
+
+  if (RUBY_VERSION =~ /^1\.9\./ || RUBY_VERSION =~ /^2\.0\./) && ((! defined?(RUBY_ENGINE)) || (RUBY_ENGINE != 'jruby'))
+    s.add_development_dependency "pry"
+    s.add_development_dependency "pry-debugger"
+    s.add_development_dependency "pry-stack_explorer"
+  end
+
+  ar_version = ENV['LOW_CARD_TABLES_AR_TEST_VERSION']
+  ar_version = ar_version.strip if ar_version
+
+  version_spec = case ar_version
+  when nil then [ ">= 3.0", "<= 4.99.99" ]
+  when 'master' then nil
+  else [ "=#{ar_version}" ]
+  end
+
+  if version_spec
+    s.add_dependency("activerecord", *version_spec)
+  end
+
+  s.add_dependency "activesupport", ">= 3.0", "<= 4.99.99"
+
+  ar_import_version = case ar_version
+  when nil then nil
+  when 'master', /^4\.0\./ then '~> 0.4.1'
+  when /^3\.0\./ then '~> 0.2.11'
+  when /^3\.1\./, /^3\.2\./ then '~> 0.3.1'
+  else raise "Don't know what activerecord-import version to require for activerecord version #{ar_version.inspect}!"
+  end
+
+  if ar_import_version
+    s.add_dependency("activerecord-import", ar_import_version)
+  else
+    s.add_dependency("activerecord-import")
+  end
+
+  require File.expand_path(File.join(File.dirname(__FILE__), 'spec', 'low_card_tables', 'helpers', 'database_helper'))
+  database_gem_name = LowCardTables::Helpers::DatabaseHelper.maybe_database_gem_name
+
+  # Ugh. Later versions of the 'mysql2' gem are incompatible with AR 3.0.x; so, here, we explicitly trap that case
+  # and use an earlier version of that Gem.
+  if database_gem_name && database_gem_name == 'mysql2' && ar_version && ar_version =~ /^3\.0\./
+    s.add_development_dependency('mysql2', '~> 0.2.0')
+  else
+    s.add_development_dependency(database_gem_name)
+  end
+end

data/spec/low_card_tables/helpers/database_helper.rb

@@ -0,0 +1,148 @@
+require 'low_card_tables/version_support'
+
+module LowCardTables
+  module Helpers
+    class DatabaseHelper
+      class InvalidDatabaseConfigurationError < StandardError; end
+
+      class << self
+        def maybe_database_gem_name
+          begin
+            dh = new
+            dh.database_gem_name
+          rescue InvalidDatabaseConfigurationError => idce
+            nil
+          end
+        end
+      end
+
+      def initialize
+        config # make sure we raise on instantiation if configuration is invalid
+      end
+
+      def setup_activerecord!
+        require 'active_record'
+        require config[:require]
+        ::ActiveRecord::Base.establish_connection(config[:config])
+
+        require 'logger'
+        require 'stringio'
+        @logs = StringIO.new
+        ::ActiveRecord::Base.logger = Logger.new(@logs)
+
+        if config[:config][:adapter] == 'sqlite3'
+          sqlite_version = ::ActiveRecord::Base.connection.send(:sqlite_version).instance_variable_get("@version").inspect rescue "unknown"
+        end
+      end
+
+      def table_name(name)
+        "lctables_spec_#{name}"
+      end
+
+      def database_gem_name
+        config[:database_gem_name]
+      end
+
+      private
+      def config
+        config_from_config_file || travis_ci_config_from_environment || invalid_config_file!
+      end
+
+      def config_from_config_file
+        return nil unless File.exist?(config_file_path)
+        require config_file_path
+
+        return nil unless defined?(LOW_CARD_TABLES_SPEC_DATABASE_CONFIG)
+        return nil unless LOW_CARD_TABLES_SPEC_DATABASE_CONFIG.kind_of?(Hash)
+
+        return nil unless LOW_CARD_TABLES_SPEC_DATABASE_CONFIG[:require]
+        return nil unless LOW_CARD_TABLES_SPEC_DATABASE_CONFIG[:database_gem_name]
+
+        return nil unless LOW_CARD_TABLES_SPEC_DATABASE_CONFIG
+        LOW_CARD_TABLES_SPEC_DATABASE_CONFIG
+      end
+
+      def travis_ci_config_from_environment
+        dbtype = (ENV['LOW_CARD_TABLES_TRAVIS_CI_DATABASE_TYPE'] || '').strip.downcase
+        is_jruby = defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
+
+        if is_jruby
+          case dbtype
+          when 'mysql'
+            {
+              :require => 'activerecord-jdbcmysql-adapter',
+              :database_gem_name => 'activerecord-jdbcmysql-adapter',
+              :config => {
+                :adapter => 'jdbcmysql',
+                :database => 'myapp_test',
+                :username => 'travis',
+                :encoding => 'utf8'
+              }
+            }
+          when '', nil then nil
+          else
+            raise "Unknown Travis CI database type: #{dbtype.inspect}"
+          end
+        else
+          case dbtype
+          when 'postgres', 'postgresql'
+            {
+              :require => 'pg',
+              :database_gem_name => 'pg',
+              :config => {
+                :adapter => 'postgresql',
+                :database => 'myapp_test',
+                :username => 'postgres',
+                :min_messages => 'WARNING'
+              }
+            }
+          when 'mysql'
+            {
+              :require => 'mysql2',
+              :database_gem_name => 'mysql2',
+              :config => {
+                :adapter => 'mysql2',
+                :database => 'myapp_test',
+                :username => 'travis',
+                :encoding => 'utf8'
+              }
+            }
+          when 'sqlite'
+            {
+              :require => 'sqlite3',
+              :database_gem_name => 'sqlite3',
+              :config => {
+                :adapter => 'sqlite3',
+                :database => ':memory:',
+                :timeout => 500
+              }
+            }
+          when '', nil then nil
+          else
+            raise "Unknown Travis CI database type: #{dbtype.inspect}"
+          end
+        end
+      end
+
+      def config_file_path
+        @config_file_path ||= File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'spec_database_config.rb'))
+      end
+
+      def invalid_config_file!
+        raise Errno::ENOENT, %{In order to run specs for LowCardTables, you need to create a file at:
+
+#{config_file_path}
+
+...that defines a top-level LOW_CARD_TABLES_SPEC_DATABASE_CONFIG hash, with members:
+
+  :require => 'name_of_adapter_to_require',
+  :database_gem_name => 'name_of_gem_for_adapter',
+  :config  => { ...whatever ActiveRecord::Base.establish_connection should be passed... }
+
+Alternatively, if you're running under Travis CI, you can set the environment variable
+LOW_CARD_TABLES_TRAVIS_CI_DATABASE_TYPE to 'postgres', 'mysql', or 'sqlite', and it will
+use the correct configuration for testing on Travis CI.}
+      end
+    end
+  end
+end

data/spec/low_card_tables/helpers/query_spy_helper.rb

@@ -0,0 +1,47 @@
+module LowCardTables
+  module Helpers
+    class QuerySpyHelper
+      class << self
+        def with_query_spy(*args, &block)
+          new(*args).spy(&block)
+        end
+      end
+
+      def initialize(table_name)
+        @table_name = table_name
+        @calls = [ ]
+      end
+
+      def spy(&block)
+        begin
+          register!
+          block.call(self)
+        ensure
+          deregister!
+        end
+      end
+
+      def call_count
+        @calls.length
+      end
+
+      def call(notification_name, when1, when2, id, data)
+        sql = data[:sql]
+        if sql && sql.strip.length > 0
+          if sql =~ /^\s*SELECT.*FROM\s+['"\`]*\s*#{@table_name}\s*['"\`]*\s+/mi
+            @calls << data.merge(:backtrace => caller)
+          end
+        end
+      end
+
+      private
+      def register!
+        ActiveSupport::Notifications.subscribe("sql.active_record", self)
+      end
+
+      def deregister!
+        ActiveSupport::Notifications.unsubscribe(self)
+      end
+    end
+  end
+end