sequel-schema-sharding 0.12.1 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 212e5cc355fae7e55ca55ccd722cf9be495fd602
-  data.tar.gz: a57f593da04a1b9e1f57d9988ecfecbb74284982
+  metadata.gz: 925ef9b402dcf173f9cd3a79524b1abe88ff9d46
+  data.tar.gz: 66794b90a0898e0ea70808311acd757eb6fcb252
 SHA512:
-  metadata.gz: a23d79950690133c3b266b0277ce70fbf8125839f404d0f9b62d7cf2e5db00609ea5ae2aad76e608d62c8d8f0cc891ecf5c2b65ddac11ad7493d327e3c090a95
-  data.tar.gz: b36fa474f6b495eda256ef016c937895fff6bee84046afbb0738f54c8db0228abaf233c4f64f86ad49c4559d26ef38bdc69cf0858371f73346a039705574af07
+  metadata.gz: a94b17459afdb2d26193fe6910c6174208aa1a3e786fc53e12783e9f5714051b167d8e714b0e6b7468d7ef1dd8c8d833d975d6f0eabaa46700c2bab809747e3d
+  data.tar.gz: de3348153fb3c7bb6e0f7e4752662a410e38c516970b5b56d1772c87bf29476d0aa42a6815888092347d888bf4fee59482d445c90d36a485cd75dd523b8c79f7

data/README.md

@@ -131,6 +131,13 @@ local database name.
 See http://sequel.rubyforge.org/rdoc/files/doc/sharding_rdoc.html for more
 information.
 
+Note that `sequel-schema-sharding` depends on the `sequel-replica-failover`
+gem. This means that when making queries to `:read_only` servers (i.e.
+replicas), certain connection errors will be rescued and re-run against
+another `:read_only` server. It may be advantageous to include each master
+database among its replicas, to ensure that read failures on a replica are
+re-run against the master.
+
 ### Models
 
 Models declare their table in the class definition. This allows Sequel
@@ -203,10 +210,14 @@ queries do not try to reconnect to a downed master.
 
 ```bash
 > bundle install
-> bundle exec rake sequel:db:test:reset
+> bundle exec rake reset
 > bundle exec rspec
 ```
 
+The default max file descriptor limit on Macs is tragically low. If tests
+fail with `too many open files`, you can run `ulimit -n 2048` to raise the
+limit.
+
 ## FAQ
 
 ### How should I shard my databases?

data/Rakefile

@@ -1,2 +1,3 @@
 require "bundler/gem_tasks"
 import 'lib/sequel/tasks/test.rake'
+import 'lib/sequel/tasks/reset.rake'

data/lib/sequel/schema-sharding.rb

@@ -10,6 +10,7 @@ require 'sequel/schema-sharding/finder'
 require 'sequel/schema-sharding/monkey_patching'
 require 'sequel/schema-sharding/model'
 require 'sequel/schema-sharding/logger_proxy'
+require 'sequel/schema-sharding/connection_strategies/random'
 
 module Sequel
   module SchemaSharding
@@ -21,6 +22,14 @@ module Sequel
       @config = config
     end
 
+    def self.replica_strategy
+      @replica_strategy ||= Sequel::SchemaSharding::ConnectionStrategy::Random
+    end
+
+    def self.replica_strategy=(strategy)
+      @replica_strategy = strategy
+    end
+
     def self.logger
       @logger ||= Logger.new(nil)
     end

data/lib/sequel/schema-sharding/connection_manager.rb

@@ -41,7 +41,6 @@ module Sequel
       # it (ex. %s, %02d).
 
       def schema_for(table_name, shard_number)
-        number_of_shards = config.number_of_shards(table_name)
         pattern = config.schema_name(table_name)
         sprintf pattern, shard_number
       end
@@ -76,15 +75,14 @@ module Sequel
 
       def replica_hash_for(config)
         return {} if config['replicas'].nil?
-        size = config['replicas'].size
         {
-          :servers => {
-            :read_only => ->(db) do
-              choice = rand(size)
+          servers: {
+            read_only: ->(db) {
+              replica_config = Sequel::SchemaSharding.replica_strategy.choose(db, config)
               probe = Sequel::SchemaSharding::DTraceProvider.provider.replica_hash_for
-              probe.fire(choice, size) if probe.enabled?
-              sequel_connection_config_for(config['replicas'][choice])
-            end
+              probe.fire(replica_config.inspect) if probe.enabled?
+              sequel_connection_config_for(replica_config)
+            }
           }
         }
       end

data/lib/sequel/schema-sharding/connection_strategies/primary_with_failover.rb

@@ -0,0 +1,21 @@
+# This strategy is used for choosing a :read_only server
+# to connect to.
+#
+# It will default to the first server in the list. In the event that
+# the first server is unavailable, the remaining replicas will be
+# randomly chosen from.
+module Sequel
+  module SchemaSharding
+    module ConnectionStrategy
+      class PrimaryWithFailover
+        def self.choose(db, config)
+          if db.pool.failing_over?
+            config['replicas'][1..-1].sample
+          else
+            config['replicas'].first
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sequel/schema-sharding/connection_strategies/random.rb

@@ -0,0 +1,16 @@
+# This strategy is used for choosing a :read_only server
+# to connect to.
+#
+# A random server will be chosen from the replica list
+# for this physical shard.
+module Sequel
+  module SchemaSharding
+    module ConnectionStrategy
+      class Random
+        def self.choose(db, config)
+          config['replicas'].sample
+        end
+      end
+    end
+  end
+end

data/lib/sequel/schema-sharding/dtrace_provider.rb

@@ -14,7 +14,7 @@ module Sequel
       end
 
       def replica_hash_for
-        @replica_hash_for ||= provider.probe(:connection_manager, :replica_hash_for, :integer, :integer)
+        @replica_hash_for ||= provider.probe(:connection_manager, :replica_hash_for, :string)
       end
 
       def shard_for

data/lib/sequel/schema-sharding/ring.rb

@@ -12,8 +12,6 @@ module Sequel
       POINTS_PER_SERVER = 1
       PRODUCTION_SHARDS = 8192
 
-      attr_accessor :shards, :continuum
-
       def initialize(shards)
         @number_of_shards = shards.size
         if ENV['RACK_ENV'] == "production"

data/lib/sequel/schema-sharding/version.rb

@@ -1,5 +1,5 @@
 module Sequel
   module SchemaSharding
-    VERSION = "0.12.1"
+    VERSION = "0.13.0"
   end
 end

data/lib/sequel/tasks/reset.rake

@@ -0,0 +1 @@
+task :reset => 'sequel:db:test:reset'

data/sequel-schema-sharding.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'sequel'
   spec.add_dependency 'pg'
-  spec.add_dependency 'sequel-replica-failover'
+  spec.add_dependency 'sequel-replica-failover', '>= 2.0.0'
   spec.add_dependency 'ruby-usdt', '>= 0.2.2'
 
   spec.add_development_dependency 'bundler', '~> 1.3'

data/spec/schema-sharding/connection_manager_spec.rb

@@ -35,7 +35,7 @@ describe Sequel::SchemaSharding::ConnectionManager do
 
   describe "#schema_for" do
     it "returns the schema name based on env and shard number" do
-      subject.schema_for('boof', 3).should eq 'sequel_logical_boof_03'
+      expect(subject.schema_for('boof', 3)).to eq 'sequel_logical_boof_03'
     end
   end
 

data/spec/schema-sharding/connection_strategies/primary_with_failover_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+require 'sequel/schema-sharding/connection_strategies/primary_with_failover'
+
+describe Sequel::SchemaSharding::ConnectionStrategy::PrimaryWithFailover do
+  let(:db) { stub(pool: stub(failing_over?: failing_over)) }
+  let(:primary) { stub }
+  let(:failover) { stub }
+  let(:config) { {'replicas' => [primary, failover]} }
+
+  subject(:strategy) { Sequel::SchemaSharding::ConnectionStrategy::PrimaryWithFailover }
+
+  describe '.choose' do
+    context 'when primary is not failing' do
+      let(:failing_over) { false }
+
+      it 'returns the primary config' do
+        expect(strategy.choose(db, config)).to eq(primary)
+      end
+    end
+
+    context 'when primary is failing' do
+      let(:failing_over) { true }
+
+      it 'returns a random server that is not primary' do
+        expect(strategy.choose(db, config)).to eq(failover)
+      end
+    end
+  end
+end

data/spec/schema-sharding/database_manager_spec.rb

@@ -25,11 +25,11 @@ describe Sequel::SchemaSharding::DatabaseManager, type: :manager, sharded: true
   describe '#create_database' do
     context 'database does not exist' do
       it 'creates the database for the current environment' do
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be false
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be false
         @manager.create_databases
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be true
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be true
       end
     end
 
@@ -40,12 +40,12 @@ describe Sequel::SchemaSharding::DatabaseManager, type: :manager, sharded: true
       end
 
       it 'outputs message to stderr' do
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be true
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be true
         $stderr.expects(:puts).with(regexp_matches(/already exists/)).twice
         @manager.create_databases
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be true
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be true
       end
     end
   end
@@ -53,23 +53,23 @@ describe Sequel::SchemaSharding::DatabaseManager, type: :manager, sharded: true
   describe '#drop_databases' do
     context 'databases exist' do
       it 'drops the database for the current environment' do
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be false
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be false
 
         @manager.create_databases
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be true
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be true
 
         @manager.drop_databases
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be false
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be false
       end
     end
 
     context 'databases dont exist' do
       it 'raises an error' do
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
-        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be false
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be false
 
         $stderr.expects(:puts).with(regexp_matches(/database doesnt exist/)).times(2)
 
@@ -85,20 +85,20 @@ describe Sequel::SchemaSharding::DatabaseManager, type: :manager, sharded: true
 
     describe '#create_shards' do
       it 'creates the database structure' do
-        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_false
-        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_false
+        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be false
+        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be false
         @manager.create_shards
-        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_true
-        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_true
+        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be true
+        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be true
       end
 
       context 'shards already exist' do
         it 'prints that shards already exist' do
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_false
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be false
           @manager.create_shards
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_true
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_true
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be true
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be true
 
           $stderr.expects(:puts).with(regexp_matches(/already exists/)).at_least_once
           @manager.create_shards
@@ -109,14 +109,14 @@ describe Sequel::SchemaSharding::DatabaseManager, type: :manager, sharded: true
     describe '#drop_schemas' do
       context 'schemas exist' do
         it 'drops the schemas' do
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_false
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be false
           @manager.create_shards
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_true
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_true
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be true
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be true
           @manager.drop_shards
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_false
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be false
         end
       end
     end

data/spec/schema-sharding/dtrace_provider_spec.rb

@@ -25,7 +25,7 @@ describe Sequel::SchemaSharding::DTraceProvider do
       end
 
       it 'takes a string argument' do
-        expect(provider.replica_hash_for.arguments).to eq([:integer, :integer])
+        expect(provider.replica_hash_for.arguments).to eq([:string])
       end
     end
 

data/spec/spec_helper.rb

@@ -8,7 +8,6 @@ require 'support/database_helper'
 require 'mocha/api'
 
 RSpec.configure do |config|
-  config.treat_symbols_as_metadata_keys_with_true_values = true
   config.run_all_when_everything_filtered = true
   config.filter_run :focus
   config.alias_example_to :fit, focus: true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sequel-schema-sharding
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.13.0
 platform: ruby
 authors:
 - Paul Henry
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-23 00:00:00.000000000 Z
+date: 2014-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -46,14 +46,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: ruby-usdt
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +122,8 @@ files:
 - lib/sequel/schema-sharding.rb
 - lib/sequel/schema-sharding/configuration.rb
 - lib/sequel/schema-sharding/connection_manager.rb
+- lib/sequel/schema-sharding/connection_strategies/primary_with_failover.rb
+- lib/sequel/schema-sharding/connection_strategies/random.rb
 - lib/sequel/schema-sharding/database_manager.rb
 - lib/sequel/schema-sharding/database_manager/schema_iterator.rb
 - lib/sequel/schema-sharding/dtrace_provider.rb
@@ -134,6 +136,7 @@ files:
 - lib/sequel/schema-sharding/monkey_patching.rb
 - lib/sequel/schema-sharding/ring.rb
 - lib/sequel/schema-sharding/version.rb
+- lib/sequel/tasks/reset.rake
 - lib/sequel/tasks/test.rake
 - sequel-schema-sharding.gemspec
 - spec/fixtures/db/migrate/artists/001_create_test_table.rb
@@ -142,6 +145,7 @@ files:
 - spec/fixtures/test_db_config.yml
 - spec/schema-sharding/configuration_spec.rb
 - spec/schema-sharding/connection_manager_spec.rb
+- spec/schema-sharding/connection_strategies/primary_with_failover_spec.rb
 - spec/schema-sharding/database_manager/schema_iterator_spec.rb
 - spec/schema-sharding/database_manager_spec.rb
 - spec/schema-sharding/dtrace_provider_spec.rb
@@ -170,7 +174,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Create horizontally sharded Sequel models with Postgres
@@ -181,6 +185,7 @@ test_files:
 - spec/fixtures/test_db_config.yml
 - spec/schema-sharding/configuration_spec.rb
 - spec/schema-sharding/connection_manager_spec.rb
+- spec/schema-sharding/connection_strategies/primary_with_failover_spec.rb
 - spec/schema-sharding/database_manager/schema_iterator_spec.rb
 - spec/schema-sharding/database_manager_spec.rb
 - spec/schema-sharding/dtrace_provider_spec.rb