sequel-schema-sharding 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9259594ee38417f3162a7b250722bb2f6d784b4b
-  data.tar.gz: 7ab7c107c400e6ec46d3cd612bda9097c82e8243
+  metadata.gz: 2c3fdaa18a8bee064030a25babedc4e261c3e160
+  data.tar.gz: 5b40cc67b014aa243ae76ef915a53cafcfc82c16
 SHA512:
-  metadata.gz: e1c5b3345a5ce05accb17231d8981a61b077cfe30f653992dbcfefb71c52689b31c1d9248ac831a2d857fb30fa326bbf383535b2ae5615600e7cc0e734316e2b
-  data.tar.gz: 47367a85fe138b51f66a8a60368ef2faf960d4c539117cc972abfc32770562cbab770a29b07cf697892d61d9d3eb07f48c3c422eca68975fa90a021bb6e61b9b
+  metadata.gz: c8bd4a45ea47e496a888abceda16c09c482402f6561c0c4e52053e8e36c1b3618af268cd8a406ac47ba86a414391b7ab42b0e2144dc8c6a8406fa4a877172877
+  data.tar.gz: d2c4673374e061cc9d9466f4f06709e8d5518daaa85a7d88465904121a78ebb5e190549ed26cda47248b4adefdcb634eeb3841201837a01dd5bb3fd82c58228e

data/README.md

@@ -35,10 +35,10 @@ conventions:
 <env>:
   tables:
     <table_name>:
-      schema_name: "schema_%e_%s"
+      schema_name: "schema_%04d"
       logical_shards:
-        <1..n>: <shard_name>
-        <n+1..m>: <shard_name>
+        <shard_name>: <1..n>
+        <shard_name>:<n+1..m>
   physical_shards:
     <shard_name>:
       host: <hostname>
@@ -49,8 +49,9 @@ conventions:
     port: <pg_port>
 ```
 
-In schema names, `%e` is replaced with the current environment, and `%s` is
-replaced by the shard number.
+In schema names `%04d` is a ```sprintf``` pattern (http://www.ruby-doc.org/core-2.0.0/Kernel.html#method-i-sprintf), where
+%d is expanded by passing the shard number. Using the pattern you can zero-pad the shard number, or use another
+pattern that suites your environment.
 
 Tables can coexist in schemas, though they do not have to.
 
@@ -142,7 +143,12 @@ the correct database connection and shard name is used. Writes will
 automatically choose the correct shard based on the sharded column.
 Never try to insert records with nil values in sharded columns.
 
-TODO: explain why we define `this`
+### this
+
+You must define a `:this` instance method on your model. `:this` must
+return a dataset that is scoped to the current instance in the database,
+so that Sequel can update/delete/etc the record when you call methods on
+the instance that persist, ie `:destroy`, `:save`.
 
 ## Running tests
 

data/examples/sharding.yml

@@ -1,12 +1,12 @@
 test:
   tables:
     artists:
-      schema_name: artists_%e_%s
+      schema_name: artists_%04d
       logical_shards:
         1..2: shard1
         3..4: shard2
     albums:
-      schema_name: albums_%e_%s
+      schema_name: albums_%04d
       logical_shards:
         1..2: shard2
         3..4: shard3
@@ -27,12 +27,12 @@ test:
 development:
   tables:
     artists:
-      schema_name: artists_%e_%s
+      schema_name: artists_%04d
       logical_shards:
         1..2: shard1
         3..4: shard2
     albums:
-      schema_name: albums_%e_%s
+      schema_name: albums_%04d
       logical_shards:
         1..2: shard2
         3..4: shard3

data/lib/sequel/schema-sharding/configuration.rb

@@ -21,15 +21,12 @@ module Sequel
         table_name = table_name.to_s
         @logical_shard_table_configs ||= {}
         @logical_shard_table_configs[table_name] ||= begin
-          table_configs = config['tables'][table_name]
-          raise "Unknown table #{table_name} in configuration" if table_configs.nil?
-          table_configs['logical_shards'].inject({}) do |hash, value|
-            eval(value[0]).each do |i|
-              hash[i] = value[1]
-            end
-            hash
-          end
+          config, number_of_shards = parse_logical_shard_config_for(table_name),
+                                     number_of_shards(table_name)
+          raise "Shard number mismatch: expected #{number_of_shards} got #{config.size} for table #{table_name}" if config.size != number_of_shards
+          config
         end
+
       end
 
       def table_names
@@ -40,8 +37,23 @@ module Sequel
         config['tables'][table_name.to_s]['schema_name']
       end
 
+      def number_of_shards(table_name)
+        config['tables'][table_name.to_s]['number_of_shards']
+      end
+
       private
 
+      def parse_logical_shard_config_for(table_name)
+        table_configs = config['tables'][table_name]
+        raise "Unknown table #{table_name} in configuration" if table_configs.nil?
+        table_configs['logical_shards'].inject({}) do |hash, value|
+          eval(value[1]).each do |i|
+            hash[i] = value[0]
+          end
+          hash
+        end
+      end
+
       def config
         yaml[env.to_s]
       end

data/lib/sequel/schema-sharding/connection_manager.rb

@@ -25,14 +25,16 @@ module Sequel
         @connections = {}
       end
 
-      def schema_for(table_name, environment, shard_number)
-        config.schema_name(table_name).gsub('%e', environment).gsub('%s', shard_number.to_s)
+      def schema_for(table_name, shard_number)
+        number_of_shards = config.number_of_shards(table_name)
+        pattern = config.schema_name(table_name)
+        sprintf pattern, shard_number
       end
 
       def default_dataset_for(table_name)
         shard_number = config.logical_shard_configs(table_name).keys.first
         shard_name = config.logical_shard_configs(table_name)[shard_number]
-        self[shard_name][:"#{schema_for(table_name, ENV['RACK_ENV'], shard_number)}__#{table_name}"]
+        self[shard_name][:"#{schema_for(table_name, shard_number)}__#{table_name}"]
       end
 
       private

data/lib/sequel/schema-sharding/database_manager.rb

@@ -57,7 +57,7 @@ module Sequel
       def create_shards
         config.table_names.each do |table_name|
           config.logical_shard_configs(table_name).each_pair do |shard_number, physical_shard|
-            schema_name = connection_manager.schema_for(table_name, env, shard_number)
+            schema_name = connection_manager.schema_for(table_name, shard_number)
             Sequel::SchemaSharding.logger.info "Creating schema #{schema_name} on #{physical_shard}.."
             connection = connection_manager.master(physical_shard)
 
@@ -81,7 +81,7 @@ module Sequel
       def drop_shards
         config.table_names.each do |table_name|
           config.logical_shard_configs(table_name).each_pair do |shard_number, physical_shard|
-            schema_name = connection_manager.schema_for(table_name, env, shard_number)
+            schema_name = connection_manager.schema_for(table_name, shard_number)
             Sequel::SchemaSharding.logger.info "Dropping schema #{schema_name} on #{physical_shard}.."
             connection = connection_manager[physical_shard]
             connection.run("DROP SCHEMA #{schema_name} CASCADE")

data/lib/sequel/schema-sharding/finder.rb

@@ -19,7 +19,7 @@ module Sequel
         physical_shard = config.logical_shard_configs(table_name)[shard_number]
 
         conn = Sequel::SchemaSharding.connection_manager[physical_shard]
-        schema = Sequel::SchemaSharding.connection_manager.schema_for(table_name, config.env, shard_number)
+        schema = Sequel::SchemaSharding.connection_manager.schema_for(table_name, shard_number)
 
         Result.new(conn, schema)
       end

data/lib/sequel/schema-sharding/ring.rb

@@ -3,6 +3,11 @@ require 'zlib'
 
 module Sequel
   module SchemaSharding
+    # This class is responsible for mapping IDs into shards, using
+    # Ketama consistent hashing algorithm, which makes it easier to
+    # rehash in the future, should the number of shards increase.
+    # For more information see http://en.wikipedia.org/wiki/Consistent_hashing
+    # This implementation is borrowed from Dali memcached library.
     class Ring
       POINTS_PER_SERVER = 1
 

data/lib/sequel/schema-sharding/version.rb

@@ -1,5 +1,5 @@
 module Sequel
   module SchemaSharding
-    VERSION = "0.2.0"
+    VERSION = "0.3.0"
   end
 end

data/spec/fixtures/test_db_config.yml

@@ -1,15 +1,17 @@
 test:
   tables:
     artists:
-      schema_name: sequel_logical_artists_%e_%s
+      schema_name: sequel_logical_artists_%02d
+      number_of_shards: 20
       logical_shards:
-        1..10: shard1
-        11..20: shard2
+        shard1:  1..10
+        shard2: 11..20
     boof:
-      schema_name: sequel_logical_boof_%e_%s
+      schema_name: sequel_logical_boof_%02d
+      number_of_shards: 20
       logical_shards:
-        1..10: shard1
-        11..20: shard2
+        shard1:  1..10
+        shard2: 11..20
   physical_shards:
     shard1:
       host: 127.0.0.1
@@ -25,15 +27,17 @@ test:
 boom:
   tables:
     artists:
-      schema_name: sequel_explosions_artists_%e_%s
+      schema_name: sequel_explosions_artists_%02d
+      number_of_shards: 20
       logical_shards:
-        1..10: shard1
-        11..20: shard2
+        shard1: "[(1..8).to_a,9,10].flatten"  # any expression that evaluates into an array works
+        shard2: 11..20
     boof:
-      schema_name: sequel_explosions_boof_%e_%s
+      schema_name: sequel_explosions_boof_%02d
+      number_of_shards: 20
       logical_shards:
-        1..10: shard1
-        11..20: shard2
+        shard1:  1..10
+        shard2: 11..20
   physical_shards:
     shard1:
       host: 127.0.0.1
@@ -47,25 +51,3 @@ boom:
     username: postgres
     password: boomboomkaboom
     port: 5432
-#boom:
-#  logical_shards:
-#    user_saves:
-#      1..10: shard1
-#      11..20: shard2
-#    product_saves:
-#      1..10: shard2
-#      11..20: shard3
-#  physical_shards:
-#    shard1:
-#      host: 127.0.0.1
-#      database: wanelo_saves_boom_shard1
-#    shard2:
-#      host: 127.0.0.1
-#      database: wanelo_saves_boom_shard2
-#    shard3:
-#      host: 127.0.0.1
-#      database: wanelo_saves_boom_shard3
-#  common:
-#    username: postgres
-#    password: boomboomkaboom
-#    port: 5432

data/spec/schema-sharding/configuration_spec.rb

@@ -19,6 +19,15 @@ describe Sequel::SchemaSharding::Configuration do
         end
       end
     end
+
+    context 'when the number of shards not the same' do
+      it 'raises an exception' do
+        config.stubs(:number_of_shards).returns(1)
+        expect {
+          config.logical_shard_configs('boof')
+        }.to raise_error(RuntimeError)
+      end
+    end
   end
 
   describe '#physical_shard_configs' do

data/spec/schema-sharding/connection_manager_spec.rb

@@ -35,7 +35,7 @@ describe Sequel::SchemaSharding::ConnectionManager do
 
   describe "#schema_for" do
     it "returns the schema name based on env and shard number" do
-      subject.schema_for('boof', 'pickles', 3).should eq 'sequel_logical_boof_pickles_3'
+      subject.schema_for('boof', 3).should eq 'sequel_logical_boof_03'
     end
   end
 
@@ -45,7 +45,7 @@ describe Sequel::SchemaSharding::ConnectionManager do
       #      This should be deconstructed to allow for injection of a mock config for testing.
       dataset = subject.default_dataset_for("artists")
       expect(dataset).to be_a(Sequel::Dataset)
-      expect(dataset.first_source_table).to eql(:'sequel_logical_artists_test_1__artists')
+      expect(dataset.first_source_table).to eql(:'sequel_logical_artists_01__artists')
     end
   end
 end

data/spec/schema-sharding/database_manager_spec.rb

@@ -85,20 +85,20 @@ describe Sequel::SchemaSharding::DatabaseManager, type: :manager, sharded: true
 
     describe '#create_shards' do
       it 'creates the database structure' do
-        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
-        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
+        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_false
+        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_false
         @manager.create_shards
-        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
-        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
+        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_true
+        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_true
       end
 
       context 'shards already exist' do
         it 'prints that shards already exist' do
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_false
           @manager.create_shards
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_true
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_true
 
           $stderr.expects(:puts).with(regexp_matches(/already exists/)).at_least_once
           @manager.create_shards
@@ -109,14 +109,14 @@ describe Sequel::SchemaSharding::DatabaseManager, type: :manager, sharded: true
     describe '#drop_schemas' do
       context 'schemas exist' do
         it 'drops the schemas' do
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_false
           @manager.create_shards
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_true
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_true
           @manager.drop_shards
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
-          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_01')).to be_false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_01')).to be_false
         end
       end
     end

data/spec/schema-sharding/finder_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'sequel/schema-sharding'
+require 'benchmark'
 
 describe Sequel::SchemaSharding::Finder do
 
@@ -7,7 +8,17 @@ describe Sequel::SchemaSharding::Finder do
     it 'returns an object with a valid connection and schema' do
       result = Sequel::SchemaSharding::Finder.instance.lookup('boof', 60)
       expect(result.connection).to be_a(Sequel::Postgres::Database)
-      expect(result.schema).to eq('sequel_logical_boof_test_2')
+      expect(result.schema).to eq('sequel_logical_boof_02')
+    end
+
+    xit 'is fast' do
+      TIMES = 150_000
+      result = Benchmark.measure do
+        TIMES.times do
+          Sequel::SchemaSharding::Finder.instance.lookup('boof', 60)
+        end
+      end
+      puts "performed #{TIMES} finder lookups: #{result}"
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sequel-schema-sharding
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Paul Henry
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-27 00:00:00.000000000 Z
+date: 2013-10-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -132,7 +132,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.7
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: Create horizontally sharded Sequel models with Postgres