low_card_tables 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +18 -0
  3. data/.travis.yml +59 -0
  4. data/Gemfile +17 -0
  5. data/LICENSE +21 -0
  6. data/README.md +75 -0
  7. data/Rakefile +6 -0
  8. data/lib/low_card_tables.rb +72 -0
  9. data/lib/low_card_tables/active_record/base.rb +55 -0
  10. data/lib/low_card_tables/active_record/migrations.rb +223 -0
  11. data/lib/low_card_tables/active_record/relation.rb +35 -0
  12. data/lib/low_card_tables/active_record/scoping.rb +87 -0
  13. data/lib/low_card_tables/errors.rb +74 -0
  14. data/lib/low_card_tables/has_low_card_table/base.rb +114 -0
  15. data/lib/low_card_tables/has_low_card_table/low_card_association.rb +273 -0
  16. data/lib/low_card_tables/has_low_card_table/low_card_associations_manager.rb +143 -0
  17. data/lib/low_card_tables/has_low_card_table/low_card_dynamic_method_manager.rb +224 -0
  18. data/lib/low_card_tables/has_low_card_table/low_card_objects_manager.rb +80 -0
  19. data/lib/low_card_tables/low_card_table/base.rb +184 -0
  20. data/lib/low_card_tables/low_card_table/cache.rb +214 -0
  21. data/lib/low_card_tables/low_card_table/cache_expiration/exponential_cache_expiration_policy.rb +151 -0
  22. data/lib/low_card_tables/low_card_table/cache_expiration/fixed_cache_expiration_policy.rb +23 -0
  23. data/lib/low_card_tables/low_card_table/cache_expiration/has_cache_expiration.rb +100 -0
  24. data/lib/low_card_tables/low_card_table/cache_expiration/no_caching_expiration_policy.rb +13 -0
  25. data/lib/low_card_tables/low_card_table/cache_expiration/unlimited_cache_expiration_policy.rb +13 -0
  26. data/lib/low_card_tables/low_card_table/row_collapser.rb +175 -0
  27. data/lib/low_card_tables/low_card_table/row_manager.rb +681 -0
  28. data/lib/low_card_tables/low_card_table/table_unique_index.rb +134 -0
  29. data/lib/low_card_tables/version.rb +4 -0
  30. data/lib/low_card_tables/version_support.rb +52 -0
  31. data/low_card_tables.gemspec +69 -0
  32. data/spec/low_card_tables/helpers/database_helper.rb +148 -0
  33. data/spec/low_card_tables/helpers/query_spy_helper.rb +47 -0
  34. data/spec/low_card_tables/helpers/system_helpers.rb +63 -0
  35. data/spec/low_card_tables/system/basic_system_spec.rb +254 -0
  36. data/spec/low_card_tables/system/bulk_system_spec.rb +334 -0
  37. data/spec/low_card_tables/system/caching_system_spec.rb +531 -0
  38. data/spec/low_card_tables/system/migrations_system_spec.rb +747 -0
  39. data/spec/low_card_tables/system/options_system_spec.rb +581 -0
  40. data/spec/low_card_tables/system/queries_system_spec.rb +142 -0
  41. data/spec/low_card_tables/system/validations_system_spec.rb +88 -0
  42. data/spec/low_card_tables/unit/active_record/base_spec.rb +53 -0
  43. data/spec/low_card_tables/unit/active_record/migrations_spec.rb +207 -0
  44. data/spec/low_card_tables/unit/active_record/relation_spec.rb +47 -0
  45. data/spec/low_card_tables/unit/active_record/scoping_spec.rb +101 -0
  46. data/spec/low_card_tables/unit/has_low_card_table/base_spec.rb +79 -0
  47. data/spec/low_card_tables/unit/has_low_card_table/low_card_association_spec.rb +287 -0
  48. data/spec/low_card_tables/unit/has_low_card_table/low_card_associations_manager_spec.rb +190 -0
  49. data/spec/low_card_tables/unit/has_low_card_table/low_card_dynamic_method_manager_spec.rb +234 -0
  50. data/spec/low_card_tables/unit/has_low_card_table/low_card_objects_manager_spec.rb +70 -0
  51. data/spec/low_card_tables/unit/low_card_table/base_spec.rb +207 -0
  52. data/spec/low_card_tables/unit/low_card_table/cache_expiration/exponential_cache_expiration_policy_spec.rb +128 -0
  53. data/spec/low_card_tables/unit/low_card_table/cache_expiration/fixed_cache_expiration_policy_spec.rb +25 -0
  54. data/spec/low_card_tables/unit/low_card_table/cache_expiration/has_cache_expiration_policy_spec.rb +100 -0
  55. data/spec/low_card_tables/unit/low_card_table/cache_expiration/no_caching_expiration_policy_spec.rb +14 -0
  56. data/spec/low_card_tables/unit/low_card_table/cache_expiration/unlimited_cache_expiration_policy_spec.rb +14 -0
  57. data/spec/low_card_tables/unit/low_card_table/cache_spec.rb +282 -0
  58. data/spec/low_card_tables/unit/low_card_table/row_collapser_spec.rb +109 -0
  59. data/spec/low_card_tables/unit/low_card_table/row_manager_spec.rb +918 -0
  60. data/spec/low_card_tables/unit/low_card_table/table_unique_index_spec.rb +117 -0
  61. metadata +206 -0
@@ -0,0 +1,134 @@
1
+ module LowCardTables
2
+ module LowCardTable
3
+ # A TableUniqueIndex represents the concept of a unique index for a given low-card model class. I say "the concept",
4
+ # because there should only be one instance of this class for any given low-card model class -- there isn't one
5
+ # instance of this class for each actual unique index for the class in question.
6
+ #
7
+ # This class started as code that was directly part of the RowManager, and was factored out to create this class
8
+ # instead -- simply so that the RowManager wouldn't have any more code in it than necessary.
9
+ class TableUniqueIndex
10
+ # Creates a new instance for the low-card model class in question.
11
+ def initialize(low_card_model)
12
+ unless low_card_model.respond_to?(:is_low_card_table?) && low_card_model.is_low_card_table?
13
+ raise ArgumentError, "You must supply a low-card AR model class, not: #{low_card_model.inspect}"
14
+ end
15
+
16
+ @low_card_model = low_card_model
17
+ end
18
+
19
+ # Ensures that the unique index is present. If the index is present, does nothing else.
20
+ #
21
+ # If the index is not present, then looks at +create_if_needed+. If this evaluates to true, then it will create
22
+ # the index. If this evaluates to false, then it will raise an exception.
23
+ def ensure_present!(create_if_needed)
24
+ return unless @low_card_model.table_exists?
25
+
26
+ current_name = current_unique_all_columns_index_name
27
+ return true if current_name
28
+
29
+ if create_if_needed
30
+ create_unique_index!
31
+ true
32
+ else
33
+ message = %{You said that the table '#{low_card_model.table_name}' is a low-card table.
34
+ However, it currently does not seem to have a unique index on all its columns. For the
35
+ low-card system to work properly, this is *required* -- although the low-card system
36
+ tries very hard to lock tables and otherwise ensure that it never will create duplicate
37
+ rows, this is important enough that we really want the database to enforce it.
38
+
39
+ We're looking for an index on the following columns:
40
+
41
+ #{value_column_names.sort.join(", ")}
42
+
43
+ ...and we have the following unique indexes:
44
+
45
+ }
46
+ current_unique_indexes.each do |unique_index|
47
+ message << " '#{unique_index.name}': #{unique_index.columns.sort.join(", ")}\n"
48
+ end
49
+ message << "\n"
50
+
51
+ raise LowCardTables::Errors::LowCardNoUniqueIndexError, message
52
+ end
53
+ end
54
+
55
+ # Removes the unique index, if one is present. If one is not present, does nothing.
56
+ def remove!
57
+ table_name = low_card_model.table_name
58
+ current_name = current_unique_all_columns_index_name
59
+
60
+ if current_name
61
+ migrate do
62
+ remove_index table_name, :name => current_name
63
+ end
64
+
65
+ now_current_name = current_unique_all_columns_index_name
66
+ if now_current_name
67
+ raise "Whoa -- we tried to remove the unique index on #{table_name}, which was named '#{current_name}', but, after we removed it, we still have a unique all-columns index called '#{now_current_name}'!"
68
+ end
69
+ end
70
+ end
71
+
72
+ private
73
+ attr_reader :low_card_model
74
+
75
+ def value_column_names
76
+ low_card_model.low_card_value_column_names
77
+ end
78
+
79
+ def migrate(&block)
80
+ migration_class = Class.new(::ActiveRecord::Migration)
81
+ metaclass = migration_class.class_eval { class << self; self; end }
82
+ metaclass.instance_eval { define_method(:up, &block) }
83
+
84
+ ::ActiveRecord::Migration.suppress_messages do
85
+ migration_class.migrate(:up)
86
+ end
87
+
88
+ low_card_model.reset_column_information
89
+ LowCardTables::VersionSupport.clear_schema_cache!(low_card_model)
90
+ end
91
+
92
+ def create_unique_index!
93
+ raise "Whoa -- there should never already be a unique index for #{low_card_model}!" if current_unique_all_columns_index_name
94
+
95
+ table_name = low_card_model.table_name
96
+ column_names = value_column_names.sort
97
+ ideal_name = ideal_unique_all_columns_index_name
98
+
99
+ migrate do
100
+ remove_index table_name, :name => ideal_name rescue nil
101
+ add_index table_name, column_names, :unique => true, :name => ideal_name
102
+ end
103
+
104
+ unless current_unique_all_columns_index_name
105
+ raise "Whoa -- there should always be a unique index by now for #{low_card_model}! We think we created one, but now it still doesn't exist?!?"
106
+ end
107
+ end
108
+
109
+ def current_unique_indexes
110
+ return [ ] if (! low_card_model.table_exists?)
111
+ low_card_model.connection.indexes(low_card_model.table_name).select { |i| i.unique }
112
+ end
113
+
114
+ def current_unique_all_columns_index_name
115
+ index = current_unique_indexes.detect { |index| index.columns.sort == value_column_names.sort }
116
+ index.name if index
117
+ end
118
+
119
+ # We just limit all index names to this length -- this should be the smallest maximum index-name length that
120
+ # any database supports.
121
+ MINIMUM_MAX_INDEX_NAME_LENGTH = 63
122
+
123
+ def ideal_unique_all_columns_index_name
124
+ index_part_1 = "index_"
125
+ index_part_2 = "_lc_on_all"
126
+
127
+ remaining_characters = MINIMUM_MAX_INDEX_NAME_LENGTH - (index_part_1.length + index_part_2.length)
128
+ index_name = index_part_1 + (@low_card_model.table_name[0..(remaining_characters - 1)]) + index_part_2
129
+
130
+ index_name
131
+ end
132
+ end
133
+ end
134
+ end
@@ -0,0 +1,4 @@
1
+ # Defines the current version of +low_card_tables+.
2
+ module LowCardTables
3
+ VERSION = "1.0.0"
4
+ end
@@ -0,0 +1,52 @@
1
+ module LowCardTables
2
+ # Contains methods used by the codebase to support differing ActiveRecord versions. This is just a clean way of
3
+ # factoring out differing ActiveRecord API into a single class.
4
+ class VersionSupport
5
+ class << self
6
+ # Clear the schema cache for a given model.
7
+ def clear_schema_cache!(model)
8
+ if model.connection.respond_to?(:schema_cache)
9
+ model.connection.schema_cache.clear!
10
+ elsif model.connection.respond_to?(:clear_cache!)
11
+ model.connection.clear_cache!
12
+ end
13
+ end
14
+
15
+ # Can you specify a block on default_scope? This was added in ActiveRecord 3.1.
16
+ def default_scopes_accept_a_block?
17
+ ! (::ActiveRecord::VERSION::MAJOR <= 3 && ::ActiveRecord::VERSION::MINOR == 0)
18
+ end
19
+
20
+ # Is #migrate a class method, or an instance method, on ActiveRecord::Migration? It changed to an instance method
21
+ # as of ActiveRecord 3.1.
22
+ def migrate_is_a_class_method?
23
+ (::ActiveRecord::VERSION::MAJOR <= 3 && ::ActiveRecord::VERSION::MINOR == 0)
24
+ end
25
+
26
+ # Define a default scope on the class in question. This is only actually used from our specs.
27
+ def define_default_scope(klass, conditions)
28
+ if default_scopes_accept_a_block?
29
+ if conditions
30
+ klass.instance_eval %{
31
+ default_scope { where(#{conditions.inspect}) }
32
+ }
33
+ else
34
+ klass.instance_eval %{
35
+ default_scope { }
36
+ }
37
+ end
38
+ else
39
+ if conditions
40
+ klass.instance_eval %{
41
+ default_scope where(#{conditions.inspect})
42
+ }
43
+ else
44
+ klass.instance_eval %{
45
+ default_scope nil
46
+ }
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,69 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "low_card_tables/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "low_card_tables"
7
+ s.version = LowCardTables::VERSION
8
+ s.authors = ["Andrew Geweke"]
9
+ s.email = ["andrew@geweke.org"]
10
+ s.homepage = "https://github.com/ageweke/low_card_tables"
11
+ s.summary = %q{"Bitfields for ActiveRecord": instead of storing multiple columns with low cardinality (few distinct values) directly in a table, which results in performance and maintainability problems, break them out into a separate table with almost zero overhead. Trivially add new columns without migrating a main, enormous table. Query on combinations of values very efficiently.}
12
+ s.description = %q{"Bitfields for ActiveRecord": store low-cardinality columns in a separate table for vastly more flexibility and better performance.}
13
+ s.license = "MIT"
14
+
15
+ s.files = `git ls-files`.split("\n")
16
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
17
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
18
+ s.require_paths = ["lib"]
19
+
20
+ s.add_development_dependency "bundler"
21
+ s.add_development_dependency "rake"
22
+ s.add_development_dependency "rspec", "~> 2.14"
23
+
24
+ if (RUBY_VERSION =~ /^1\.9\./ || RUBY_VERSION =~ /^2\.0\./) && ((! defined?(RUBY_ENGINE)) || (RUBY_ENGINE != 'jruby'))
25
+ s.add_development_dependency "pry"
26
+ s.add_development_dependency "pry-debugger"
27
+ s.add_development_dependency "pry-stack_explorer"
28
+ end
29
+
30
+ ar_version = ENV['LOW_CARD_TABLES_AR_TEST_VERSION']
31
+ ar_version = ar_version.strip if ar_version
32
+
33
+ version_spec = case ar_version
34
+ when nil then [ ">= 3.0", "<= 4.99.99" ]
35
+ when 'master' then nil
36
+ else [ "=#{ar_version}" ]
37
+ end
38
+
39
+ if version_spec
40
+ s.add_dependency("activerecord", *version_spec)
41
+ end
42
+
43
+ s.add_dependency "activesupport", ">= 3.0", "<= 4.99.99"
44
+
45
+ ar_import_version = case ar_version
46
+ when nil then nil
47
+ when 'master', /^4\.0\./ then '~> 0.4.1'
48
+ when /^3\.0\./ then '~> 0.2.11'
49
+ when /^3\.1\./, /^3\.2\./ then '~> 0.3.1'
50
+ else raise "Don't know what activerecord-import version to require for activerecord version #{ar_version.inspect}!"
51
+ end
52
+
53
+ if ar_import_version
54
+ s.add_dependency("activerecord-import", ar_import_version)
55
+ else
56
+ s.add_dependency("activerecord-import")
57
+ end
58
+
59
+ require File.expand_path(File.join(File.dirname(__FILE__), 'spec', 'low_card_tables', 'helpers', 'database_helper'))
60
+ database_gem_name = LowCardTables::Helpers::DatabaseHelper.maybe_database_gem_name
61
+
62
+ # Ugh. Later versions of the 'mysql2' gem are incompatible with AR 3.0.x; so, here, we explicitly trap that case
63
+ # and use an earlier version of that Gem.
64
+ if database_gem_name && database_gem_name == 'mysql2' && ar_version && ar_version =~ /^3\.0\./
65
+ s.add_development_dependency('mysql2', '~> 0.2.0')
66
+ else
67
+ s.add_development_dependency(database_gem_name)
68
+ end
69
+ end
@@ -0,0 +1,148 @@
1
+ require 'low_card_tables/version_support'
2
+
3
+ module LowCardTables
4
+ module Helpers
5
+ class DatabaseHelper
6
+ class InvalidDatabaseConfigurationError < StandardError; end
7
+
8
+ class << self
9
+ def maybe_database_gem_name
10
+ begin
11
+ dh = new
12
+ dh.database_gem_name
13
+ rescue InvalidDatabaseConfigurationError => idce
14
+ nil
15
+ end
16
+ end
17
+ end
18
+
19
+ def initialize
20
+ config # make sure we raise on instantiation if configuration is invalid
21
+ end
22
+
23
+ def setup_activerecord!
24
+ require 'active_record'
25
+ require config[:require]
26
+ ::ActiveRecord::Base.establish_connection(config[:config])
27
+
28
+ require 'logger'
29
+ require 'stringio'
30
+ @logs = StringIO.new
31
+ ::ActiveRecord::Base.logger = Logger.new(@logs)
32
+
33
+ if config[:config][:adapter] == 'sqlite3'
34
+ sqlite_version = ::ActiveRecord::Base.connection.send(:sqlite_version).instance_variable_get("@version").inspect rescue "unknown"
35
+ end
36
+ end
37
+
38
+ def table_name(name)
39
+ "lctables_spec_#{name}"
40
+ end
41
+
42
+ def database_gem_name
43
+ config[:database_gem_name]
44
+ end
45
+
46
+ private
47
+ def config
48
+ config_from_config_file || travis_ci_config_from_environment || invalid_config_file!
49
+ end
50
+
51
+ def config_from_config_file
52
+ return nil unless File.exist?(config_file_path)
53
+ require config_file_path
54
+
55
+ return nil unless defined?(LOW_CARD_TABLES_SPEC_DATABASE_CONFIG)
56
+ return nil unless LOW_CARD_TABLES_SPEC_DATABASE_CONFIG.kind_of?(Hash)
57
+
58
+ return nil unless LOW_CARD_TABLES_SPEC_DATABASE_CONFIG[:require]
59
+ return nil unless LOW_CARD_TABLES_SPEC_DATABASE_CONFIG[:database_gem_name]
60
+
61
+ return nil unless LOW_CARD_TABLES_SPEC_DATABASE_CONFIG
62
+ LOW_CARD_TABLES_SPEC_DATABASE_CONFIG
63
+ end
64
+
65
+ def travis_ci_config_from_environment
66
+ dbtype = (ENV['LOW_CARD_TABLES_TRAVIS_CI_DATABASE_TYPE'] || '').strip.downcase
67
+ is_jruby = defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
68
+
69
+ if is_jruby
70
+ case dbtype
71
+ when 'mysql'
72
+ {
73
+ :require => 'activerecord-jdbcmysql-adapter',
74
+ :database_gem_name => 'activerecord-jdbcmysql-adapter',
75
+ :config => {
76
+ :adapter => 'jdbcmysql',
77
+ :database => 'myapp_test',
78
+ :username => 'travis',
79
+ :encoding => 'utf8'
80
+ }
81
+ }
82
+ when '', nil then nil
83
+ else
84
+ raise "Unknown Travis CI database type: #{dbtype.inspect}"
85
+ end
86
+ else
87
+ case dbtype
88
+ when 'postgres', 'postgresql'
89
+ {
90
+ :require => 'pg',
91
+ :database_gem_name => 'pg',
92
+ :config => {
93
+ :adapter => 'postgresql',
94
+ :database => 'myapp_test',
95
+ :username => 'postgres',
96
+ :min_messages => 'WARNING'
97
+ }
98
+ }
99
+ when 'mysql'
100
+ {
101
+ :require => 'mysql2',
102
+ :database_gem_name => 'mysql2',
103
+ :config => {
104
+ :adapter => 'mysql2',
105
+ :database => 'myapp_test',
106
+ :username => 'travis',
107
+ :encoding => 'utf8'
108
+ }
109
+ }
110
+ when 'sqlite'
111
+ {
112
+ :require => 'sqlite3',
113
+ :database_gem_name => 'sqlite3',
114
+ :config => {
115
+ :adapter => 'sqlite3',
116
+ :database => ':memory:',
117
+ :timeout => 500
118
+ }
119
+ }
120
+ when '', nil then nil
121
+ else
122
+ raise "Unknown Travis CI database type: #{dbtype.inspect}"
123
+ end
124
+ end
125
+ end
126
+
127
+ def config_file_path
128
+ @config_file_path ||= File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'spec_database_config.rb'))
129
+ end
130
+
131
+ def invalid_config_file!
132
+ raise Errno::ENOENT, %{In order to run specs for LowCardTables, you need to create a file at:
133
+
134
+ #{config_file_path}
135
+
136
+ ...that defines a top-level LOW_CARD_TABLES_SPEC_DATABASE_CONFIG hash, with members:
137
+
138
+ :require => 'name_of_adapter_to_require',
139
+ :database_gem_name => 'name_of_gem_for_adapter',
140
+ :config => { ...whatever ActiveRecord::Base.establish_connection should be passed... }
141
+
142
+ Alternatively, if you're running under Travis CI, you can set the environment variable
143
+ LOW_CARD_TABLES_TRAVIS_CI_DATABASE_TYPE to 'postgres', 'mysql', or 'sqlite', and it will
144
+ use the correct configuration for testing on Travis CI.}
145
+ end
146
+ end
147
+ end
148
+ end
@@ -0,0 +1,47 @@
1
+ module LowCardTables
2
+ module Helpers
3
+ class QuerySpyHelper
4
+ class << self
5
+ def with_query_spy(*args, &block)
6
+ new(*args).spy(&block)
7
+ end
8
+ end
9
+
10
+ def initialize(table_name)
11
+ @table_name = table_name
12
+ @calls = [ ]
13
+ end
14
+
15
+ def spy(&block)
16
+ begin
17
+ register!
18
+ block.call(self)
19
+ ensure
20
+ deregister!
21
+ end
22
+ end
23
+
24
+ def call_count
25
+ @calls.length
26
+ end
27
+
28
+ def call(notification_name, when1, when2, id, data)
29
+ sql = data[:sql]
30
+ if sql && sql.strip.length > 0
31
+ if sql =~ /^\s*SELECT.*FROM\s+['"\`]*\s*#{@table_name}\s*['"\`]*\s+/mi
32
+ @calls << data.merge(:backtrace => caller)
33
+ end
34
+ end
35
+ end
36
+
37
+ private
38
+ def register!
39
+ ActiveSupport::Notifications.subscribe("sql.active_record", self)
40
+ end
41
+
42
+ def deregister!
43
+ ActiveSupport::Notifications.unsubscribe(self)
44
+ end
45
+ end
46
+ end
47
+ end