activerecord-turntable 3.1.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 66d31edc8c786999ddb71e35bf5f64e3dd97978a
-  data.tar.gz: 5cad32e83086caff4e7fdb1ad8c89e0393286c3e
+SHA256:
+  metadata.gz: 5bd7f48754157e32eaaf254a518645b9d1cde4e31840dba63b927886bedb74ee
+  data.tar.gz: 529f08173f847218298a34ed9832163e1d129b003e8ba0ef201ca63d8333b029
 SHA512:
-  metadata.gz: 5ec384d210033d0f2831b9db8381d422a913d8d2d736ecb1422b8870ee12624ff1f0eaadf46b13c3191e2c0f5d28bf6b8fef9ea9c83deaa54866e680ccd8d441
-  data.tar.gz: 67c6d75b06df8ea9b3c5014d43454f3ec4ef7411123ce76b93fff053e37a2caf60ded5eea3f9eff31bc3d2e5f503514226ef82cc1180e0a01e812df05f3fe075
+  metadata.gz: 515cc5103600046d62ea3b91a5695787a2f34d423da084e15aa3dbef8accdba238abd4bb4fb4c4e2659cdbe9ac965f020afb2d6f75a84c4e097cff28e2f1d6cb
+  data.tar.gz: '0578e2aff069a944a6b512b04ae19d47fe2591813f9666f7324cb7d592a4febaa61b457dd71b75dfe8b592f1666bdf57880a3f552da0702c1362b64126db401c'

data/.gitignore

@@ -1,6 +1,7 @@
 *.gem
 .bundle
 Gemfile.lock
+vendor/bundle
 pkg/*
 .DS_Store
 doc

data/.travis.yml

@@ -5,9 +5,10 @@ cache:
   bundler: true
 
 rvm:
-  - 2.2.7
-  - 2.3.4
-  - 2.4.1
+  - 2.2.9
+  - 2.3.6
+  - 2.4.3
+  - 2.5.0
   - ruby-head
 
 gemfile:
@@ -15,15 +16,27 @@ gemfile:
   - gemfiles/rails5_0_1.gemfile
   - gemfiles/rails5_0_2.gemfile
   - gemfiles/rails5_0_3.gemfile
+  - gemfiles/rails5_0_4.gemfile
+  - gemfiles/rails5_0_5.gemfile
   - gemfiles/rails5_1_0.gemfile
   - gemfiles/rails5_1_1.gemfile
+  - gemfiles/rails5_1_2.gemfile
+  - gemfiles/rails5_1_3.gemfile
+  - gemfiles/rails5_1_4.gemfile
+  - gemfiles/rails5_1_5.gemfile
   - gemfiles/rails_edge.gemfile
 
 env:
   - SETUP_TASK=turntable:db:reset BUILD_TASK=spec
+  - SETUP_TASK=turntable:db:reset BUILD_TASK=spec SPEC_OPTS="--tag with_katsubushi"
   - SETUP_TASK=turntable:activerecord:setup BUILD_TASK=turntable:activerecord:test
 
-before_install: gem update --system
+services:
+  - docker
+
+before_install:
+  - docker run -d --name turntable-katsubushi -p 11212:11212 katsubushi/katsubushi:latest -worker-id 1
+  - gem update --system
 
 before_script:
   - bundle exec rake $SETUP_TASK
@@ -35,14 +48,17 @@ matrix:
   exclude:
     - rvm: ruby-head
     - gemfile: gemfiles/rails_edge.gemfile
-    - rvm: 2.3.4
+    - rvm: 2.3.6
+    - rvm: 2.4.3
   include:
-    - rvm: ruby-head
+    - rvm: 2.3.6
+      gemfile: gemfiles/rails5_1_5.gemfile
+    - rvm: 2.4.3
+      gemfile: gemfiles/rails5_1_5.gemfile
+    - rvm: 2.5.0
       gemfile: gemfiles/rails_edge.gemfile
-    - rvm: 2.4.1
+    - rvm: ruby-head
       gemfile: gemfiles/rails_edge.gemfile
-    - rvm: 2.3.4
-      gemfile: gemfiles/rails5_1_1.gemfile
   allow_failures:
     - rvm: ruby-head
     - gemfile: gemfiles/rails_edge.gemfile

data/CHANGELOG.md

@@ -1,3 +1,28 @@
+## activerecord-turntable 4.0.0 ##
+
+### Major Changes
+
+* Supported rails versions are
+  * 5.0.0 to 5.0.5
+  * 5.1.0 to 5.1.5
+* Configuration
+  * Added DSL configuration file: config/turntable.rb
+* Added a new algorithm: `hash_slot`
+  * distributes like redis cluster.
+* Added a new sequencer: `katsubushi`
+  * Support [katsubushi](https://github.com/kayac/go-katsubushi) as a sequencer backend.
+* Support slave(read replica) connection. (Experimental feature)
+
+### Incompatible changes
+
+* `RangeAlgorithm` is integrated to `RangeBsearchAlgorithm`.
+* Changed `ConnectionProxy#with_master` behavior to `Fix connection to primary master database`.
+  * Old `ConnectionProxy#with_master` behavior(default model connection) is renamed to `ConnectionProxy#with_default_shard`
+
+### Internal Change
+
+* Changed `AR::Base.turntable_configuration` to use `ActiveRecord::Turntable::Configuration` class instead of Hash.
+
 ## activerecord-turntable 3.1.0 ##
 
 ### Major Changes

data/Gemfile

@@ -4,3 +4,5 @@ gemspec
 
 gem "activerecord", "~> 5.1.1"
 gem "activesupport", "~> 5.1.1"
+
+gem "mysql2", "~> 0.4.4"

data/Guardfile

@@ -2,7 +2,7 @@
 # More info at https://github.com/guard/guard#readme
 
 guard "rspec",
-      cmd: "rspec",
+      cmd: "bundle exec rspec",
       all_after_pass: true,
       all_on_start: true do
   watch(%r{^spec/.+_spec\.rb$})

data/README.md

@@ -2,8 +2,8 @@
 
 [![Gem Version](https://badge.fury.io/rb/activerecord-turntable.svg)](http://badge.fury.io/rb/activerecord-turntable)
 [![Build Status](https://travis-ci.org/drecom/activerecord-turntable.svg?branch=master)](https://travis-ci.org/drecom/activerecord-turntable)
-[![Dependency Status](https://gemnasium.com/drecom/activerecord-turntable.svg)](https://gemnasium.com/drecom/activerecord-turntable)
-[![Coverage Status](https://coveralls.io/repos/drecom/activerecord-turntable/badge.png?branch=master)](https://coveralls.io/r/drecom/activerecord-turntable?branch=master)
+[![Coverage Status](https://coveralls.io/repos/github/drecom/activerecord-turntable/badge.svg)](https://coveralls.io/github/drecom/activerecord-turntable)
+[![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/drecom/activerecord-turntable/blob/master/LICENSE.txt)
 
 ActiveRecord::Turntable is a database sharding extension for ActiveRecord.
 
@@ -11,21 +11,23 @@ ActiveRecord::Turntable is a database sharding extension for ActiveRecord.
 
 activerecord(>=5.0.0, <6.0)
 
+* Current latest stable branch is [stable branch](https://github.com/drecom/activerecord-turntable/tree/stable).
+
 If you are using with older activerecord versions, use following versions.
 
 * activerecord 4.x - use activerecord-turntable version 2.x.([stable-2-x branch](https://github.com/drecom/activerecord-turntable/tree/stable-2-x))
 * activerecord 3.x - use activerecord-turntable version 1.x.([stable-1-x branch](https://github.com/drecom/activerecord-turntable/tree/stable-1-x))
 
-## Supported Database
+## Supported DBMS
 
-Currently supports mysql only.
+MySQL only.
 
 ## Installation
 
 Add to Gemfile:
 
 ```ruby
-gem 'activerecord-turntable', '~> 3.0.0'
+gem 'activerecord-turntable', '~> 4.0.0'
 ```
 
 Run a bundle install:
@@ -85,7 +87,33 @@ three user databases sharded by user_id.
 
 ### Example Configuration
 
-Edit turntable.yml and database.yml. See below example config.
+Edit DSL type configuration file(config/turntable.rb) or YAML configuration file(turntable.yml).
+And add database connection settings to database.yml.
+
+See below example config.
+
+* example turntable.rb
+
+```ruby
+cluster :user_cluster do
+  # algorithm [algorithm name symbol(range|range_bsearch|modulo)]
+  algorithm :range_bsearch
+
+  # sequencer [sequence name] [sequence type] [*options hash]
+  sequencer :user_seq, :mysql, connection: :user_seq
+
+  # shard [range], to: [connection names in database.yml]
+  shard   1...100,        to: :user_shard_1
+  shard 100...200,        to: :user_shard_2
+  shard 200...2000000000, to: :user_shard_3
+
+  # If you are using modulo algorithm, pass integer sequence start with zero.
+  #
+  # shard 0, to: :user_shard_1
+  # shard 1, to: :user_shard_2
+  # shard 2, to: :user_shard_3
+end
+```
 
 * example turntable.yml
 
@@ -139,6 +167,124 @@ Edit turntable.yml and database.yml. See below example config.
           database: sample_app_user3_development
 ```
 
+#### About algorithms
+
+* range, range_bsearch - distribute records by key range
+
+```ruby
+  algorithm :range_bsearch
+
+  shard      1...20_000,     to: :user_shard_1
+  shard 20_000...40_000,     to: :user_shard_2
+  shard 40_000...60_000,     to: :user_shard_1
+  shard 60_000...80_000,     to: :user_shard_2
+  shard 80_000...10_000_000, to: :user_shard_3
+```
+
+* modulo - distribute records by key modulo
+
+```ruby
+  algorithm :modulo
+
+  shard 0, to: :user_shard_1
+  shard 1, to: :user_shard_2
+  shard 2, to: :user_shard_3
+```
+
+* hash_slot - distribute records by key hashes
+
+default hash function is `Zlib.crc32(key.to_s)`
+
+```ruby
+  algorithm :hash_slot
+  # Or specify hash function
+  # algorithm :hash_slot, hash_func: ->(key) { Zlib.adler32(key.to_s) }
+
+  shard     0...4096,  to: :user_shard_1
+  shard  4096...8192,  to: :user_shard_2
+  shard  8192...12288, to: :user_shard_3
+  shard 12288...16384, to: :user_shard_4
+```
+
+### Slave support (experimental)
+
+Slave enabled configuration examples:
+
+* config/turntable.rb
+
+```ruby
+cluster :user_cluster do
+...
+
+  # shard [range], to: [connection names in database.yml]
+  shard   1...100,        to: :user_shard_1, slaves: [:user_shard_1_1]
+  shard 100...200,        to: :user_shard_2, slaves: [:user_shard_2_1]
+  shard 200...2000000000, to: :user_shard_3, slaves: [:user_shard_3_1]
+end
+```
+
+* config/turntable.yml
+
+```yaml
+    development:
+      clusters:
+        user_cluster: # <-- cluster name
+          ...
+          shards:
+            - connection: user_shard_1
+              less_than: 100
+              slaves:
+                - user_shard_1_1
+            - connection: user_shard_2
+              less_than: 200
+              slaves:
+                - user_shard_2_1
+            - connection: user_shard_3
+              less_than: 2000000000
+              slaves:
+                - user_shard_3_1
+```
+
+* config/database.yml
+
+Add slave connection settings under `shards`.
+
+```yaml
+...
+  shards:
+    user_shard_1:
+      <<: *default
+      database: turntable_user_shard_1_test
+    user_shard_1_1:
+      <<: *default
+      database: turntable_user_shard_1_1_test
+    user_shard_2:
+      <<: *default
+      database: turntable_user_shard_2_test
+    user_shard_2_1:
+      <<: *default
+      database: turntable_user_shard_2_1_test
+    user_shard_3:
+      <<: *default
+      database: turntable_user_shard_3_test
+    user_shard_3_1:
+      <<: *default
+      database: turntable_user_shard_3_1_test
+```
+
+Slave usage:
+
+```ruby
+User.with_slave {
+  # `User` model will use slave databases within this block.
+}
+
+User.with_master {
+  # `User` model will use master database within this block.
+}
+```
+
+
 ### Example Migration
 
 Generate a model:
@@ -255,10 +401,11 @@ end
 
 Sequencer provides generating global IDs.
 
-Turntable has follow 2 sequencers currently:
+Turntable has follow 3 sequencers currently:
 
 * :mysql - Use database table to generate ids.
 * :barrage - Use [barrage](https://github.com/drecom/barrage) gem to generate ids
+* :katsubushi - [katsubushi](https://github.com/kayac/go-katsubushi) sequencer backend
 
 ### Mysql example
 
@@ -348,6 +495,47 @@ Next, add sequencer definition to the model:
   end
 ```
 
+### Katsubushi example
+
+[katsubushi](https://github.com/kayac/go-katsubushi) is available as a sequence server.
+ActiveRecord::Turntable accesses katsubushi via Dalli.
+
+First, add dalli gem to your Gemfile:
+
+```ruby
+gem 'dalli'
+```
+
+Then, add configuration to turntable.yml:
+
+* turntable.yml
+
+```yaml
+    development:
+      clusters:
+        user_cluster: # <-- cluster name
+          ....
+          seq:
+            katsubushi_seq: # <-- sequencer name
+              seq_type: katsubushi # <-- sequencer type
+              options: # <-- options passed to dalli
+                servers:
+                  - host: localhost
+                    port: 11212
+                  - host: localhost
+                    port: 11213
+```
+
+Next, add sequencer definition to the model:
+
+```ruby
+  class User < ApplicationRecord
+    turntable :id
+    sequencer :katsubushi_seq # <-- this line enables sequencer module
+    has_one :status
+  end
+```
+
 ## Transactions
 Turntable has some transaction support methods.
 
@@ -497,7 +685,7 @@ To notice queries causing performance problem, Turntable has follow options.
 * raise\_on\_not\_specified\_shard\_update - raises on updates executed on all shards
 
 
-Add to turntable.yml:
+Add to turntable.yml or turntable.rb:
 
 ```yaml
 development:
@@ -506,6 +694,12 @@ development:
    raise_on_not_specified_shard_update: true
 ```
 
+```ruby
+# Write on top level
+raise_on_not_specified_shard_query true
+raise_on_not_specified_shard_update true
+```
+
 ## Thanks
 
 ConnectionProxy, Distributed Migration implementation is inspired by Octopus and DataFabric.

data/Rakefile

@@ -47,73 +47,7 @@ namespace :turntable do
 
       configurations.each do |configuration|
         ActiveRecord::Base.establish_connection configuration
-
-        ActiveRecord::Base.connection.create_table :users, comment: "comment" do |t|
-          t.string :nickname
-          t.string :thumbnail_url
-          t.binary :blob
-          t.datetime :joined_at
-          t.datetime :deleted_at
-          t.timestamps
-        end
-        ActiveRecord::Base.connection.create_sequence_for :users, comment: "comment"
-
-        ActiveRecord::Base.connection.create_table :user_statuses do |t|
-          t.belongs_to :user, :null => false
-          t.integer    :hp,   :null => false, :default => 0
-          t.integer    :mp,   :null => false, :default => 0
-          t.text       :data
-          t.integer    :lock_version, :null => false, :default => 0
-          t.datetime   :deleted_at, :default => nil
-          t.timestamps
-        end
-        ActiveRecord::Base.connection.create_sequence_for :user_statuses
-
-        ActiveRecord::Base.connection.create_table :cards do |t|
-          t.string :name, :null => false
-          t.integer :hp,  :null => false, :default => 0
-          t.integer :mp,  :null => false, :default => 0
-          t.timestamps
-        end
-        ActiveRecord::Base.connection.create_table :archived_cards do |t|
-          t.string :name, :null => false
-          t.integer :hp,  :null => false, :default => 0
-          t.integer :mp,  :null => false, :default => 0
-          t.timestamps
-          t.datetime :deleted_at, :default => nil
-        end
-
-        ActiveRecord::Base.connection.create_table :cards_users do |t|
-          t.belongs_to :card,    :null => false
-          t.belongs_to :user,    :null => false
-          t.integer    :num,     :default => 1, :null => false
-          t.timestamps
-        end
-        ActiveRecord::Base.connection.create_sequence_for :cards_users
-
-        ActiveRecord::Base.connection.create_table :archived_cards_users do |t|
-          t.belongs_to :card,    :null => false
-          t.belongs_to :user,    :null => false
-          t.timestamps
-          t.datetime   :deleted_at, :default => nil
-        end
-        ActiveRecord::Base.connection.create_sequence_for :archived_cards_users
-
-        ActiveRecord::Base.connection.create_table :cards_users_histories do |t|
-          t.belongs_to :cards_user,    :null => false
-          t.belongs_to :user, :null => false
-          t.timestamps
-        end
-        ActiveRecord::Base.connection.create_sequence_for :cards_users_histories
-
-        ActiveRecord::Base.connection.create_table :events_users_histories do |t|
-          t.belongs_to :events_user, :null => false
-          t.belongs_to :cards_user,    :null => false
-          t.belongs_to :user, :null => false
-          t.string :type, :default => nil
-          t.timestamps
-        end
-        ActiveRecord::Base.connection.create_sequence_for :events_users_histories
+        load File.expand_path("spec/migrations/schema.rb", __dir__)
       end
     end
 
@@ -170,6 +104,47 @@ namespace :turntable do
             EOS
           end
         end
+
+        # Ignore failing migrator test
+        if ActiveRecord.gem_version.release <= Gem::Version.new("5.1.5")
+          File.open("test/cases/migrator_test.rb", "a") do |f|
+            f << <<-EOS.strip_heredoc
+              class MigratorTest
+                undef_method :test_migrator_verbosity if method_defined?(:test_migrator_verbosity)
+              end
+            EOS
+          end
+        end
+
+        # Ignore some failing tests with ruby 2.5
+        if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.5") &&
+           ActiveRecord.gem_version.release <= Gem::Version.new("5.1.4")
+          ignores = [
+            ["aggregations_test.rb", "AggregationsTest", ["test_immutable_value_objects"]],
+            ["query_cache_test.rb",  "QueryCacheTest", ["test_query_cache_does_not_allow_sql_key_mutation"]],
+            ["transactions_test.rb", "TransactionTest", ["test_rollback_when_saving_a_frozen_record"]],
+            ["log_subscriber_test.rb", "LogSubscriberTest",
+             %w[test_basic_payload_name_logging_coloration_generic_sql
+                test_basic_payload_name_logging_coloration_named_sql
+                test_query_logging_coloration_with_nested_select
+                test_query_logging_coloration_with_multi_line_nested_select
+                test_query_logging_coloration_with_lock]],
+          ]
+
+          ignores.each do |file_name, class_name, method_names|
+            path = File.join("test/cases", file_name)
+            next unless File.exist?(path)
+
+            File.open(File.join("test/cases", file_name), "a") do |f|
+              f << <<-EOS.strip_heredoc
+                class #{class_name}
+                  #{method_names.map { |method_name| "undef_method :#{method_name} if method_defined?(:#{method_name})" }.join("\n") }
+
+                end
+              EOS
+            end
+          end
+        end
       end
 
       task :db => :rails do