sequel-schema-sharding 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9a83484015fdcd8dc4799e752d6d82ee94498738
+  data.tar.gz: 2e44e0d65a6c598371035daddc1724022b3c893b
+SHA512:
+  metadata.gz: bac9d7a0ed39dbe558e2f0bd6422bda34f68676662ba5283722340c1ffbf784f6e8c433ca6b118efe41c8a0e66ed123c8bfd2f482597ace1ab750a2b7d2fcbd1
+  data.tar.gz: 724cae47d487ee6e63a8e969ae8ed9f1d380ffaf61b3877074578a9704565fd0290ffd27550904afc9ad3949aba536b4fa69d5209670a39b13f21ae2e0fd874c

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0-p247
+before_script: "bundle exec rake sequel:db:create"
+script: "bundle exec rspec"
+notifications:
+  email: false

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sequel-sharding.gemspec
+gemspec
+
+group :test do
+  gem 'rspec'
+  gem 'mocha', require: false
+  gem 'pry-nav'
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 TODO: Write your name
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,33 @@
+Sequel::SchemaSharding
+================
+
+[![Build Status](https://travis-ci.org/wanelo/sequel-sharding.png?branch=master)](https://travis-ci.org/wanelo/sequel-sharding)
+
+Horizontally shard postgres with the Sequel gem. This gem allows you to configure mappings between logical and
+physical shards, and pool connections between logical shards on the same physical server.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sequel-sharding'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sequel-sharding
+
+## Usage
+
+TODO :)
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+import 'lib/sequel/tasks/test.rake'

data/lib/sequel-schema-sharding.rb

@@ -0,0 +1 @@
+require 'sequel/schema-sharding'

data/lib/sequel/schema-sharding.rb

@@ -0,0 +1,54 @@
+require 'sequel/schema-sharding/version'
+require 'sequel/schema-sharding/configuration'
+require 'sequel/schema-sharding/connection_manager'
+require 'sequel/schema-sharding/database_manager'
+require 'sequel/schema-sharding/ring'
+require 'sequel/schema-sharding/finder'
+require 'sequel/schema-sharding/sequel_ext'
+require 'sequel/schema-sharding/model'
+require 'logger'
+
+module Sequel
+  module SchemaSharding
+    def self.config
+      @config ||= Sequel::SchemaSharding::Configuration.new(ENV['RACK_ENV'], sharding_yml_path)
+    end
+
+    def self.config=(config)
+      @config = config
+    end
+
+    def self.logger
+      @logger ||= Logger.new($stdout)
+    end
+
+    def self.logger=(logger)
+      @logger = logger
+    end
+
+    def self.connection_manager
+      @connection_manager ||= ConnectionManager.new
+    end
+
+    def self.connection_manager=(connection_manager)
+      @connection_manager = connection_manager
+    end
+
+    def self.sharding_yml_path
+      @sharding_yml_path ||= File.expand_path('../../../config/sharding.yml', __FILE__)
+    end
+
+    def self.sharding_yml_path=(path)
+      @sharding_yml_path = path
+    end
+
+    def self.migration_path
+      @migration_path || raise('You must set the migration path.')
+    end
+
+    def self.migration_path=(path)
+      @migration_path = path
+    end
+
+  end
+end

data/lib/sequel/schema-sharding/configuration.rb

@@ -0,0 +1,54 @@
+require 'yaml'
+
+module Sequel
+  module SchemaSharding
+    class Configuration
+      attr_reader :env, :yaml_path
+
+      def initialize(env, yaml_path)
+        @env = env
+        @yaml_path = yaml_path
+      end
+
+      def physical_shard_configs
+        @physical_shard_configs ||= config['physical_shards'].inject({}) do |hash, value|
+          hash[value[0]] = config['common'].merge(value[1])
+          hash
+        end
+      end
+
+      def logical_shard_configs(table_name)
+        table_name = table_name.to_s
+        @logical_shard_table_configs ||= {}
+        @logical_shard_table_configs[table_name] ||= begin
+          table_configs = config['tables'][table_name]
+          raise "Unknown table #{table_name} in configuration" if table_configs.nil?
+          table_configs['logical_shards'].inject({}) do |hash, value|
+            eval(value[0]).each do |i|
+              hash[i] = value[1]
+            end
+            hash
+          end
+        end
+      end
+
+      def table_names
+        config['tables'].keys
+      end
+
+      def schema_name(table_name)
+        config['tables'][table_name.to_s]['schema_name']
+      end
+
+      private
+
+      def config
+        yaml[env.to_s]
+      end
+
+      def yaml
+        @raw_yaml ||= YAML.load_file(yaml_path)
+      end
+    end
+  end
+end

data/lib/sequel/schema-sharding/connection_manager.rb

@@ -0,0 +1,48 @@
+require 'singleton'
+
+module Sequel
+  module SchemaSharding
+    class ConnectionManager
+      attr_reader :connections
+
+      def initialize
+        @connections = {}
+      end
+
+      def [](name)
+        config = db_config_for(name)
+        @connections[name.to_s] ||= Sequel.postgres(:user => config['username'],
+                                                    :password => config['password'],
+                                                    :host => config['host'],
+                                                    :database => config['database'])
+      end
+
+      def disconnect
+        @connections.each_value do |conn|
+          conn.disconnect
+        end
+        @connections = {}
+      end
+
+      def schema_for(table_name, environment, shard_number)
+        config.schema_name(table_name).gsub('%e', environment).gsub('%s', shard_number.to_s)
+      end
+
+      def default_dataset_for(table_name)
+        shard_number = config.logical_shard_configs(table_name).keys.first
+        shard_name = config.logical_shard_configs(table_name)[shard_number]
+        self[shard_name][:"#{schema_for(table_name, ENV['RACK_ENV'], shard_number)}__#{table_name}"]
+      end
+
+      private
+
+      def db_config_for(name)
+        config.physical_shard_configs[name]
+      end
+
+      def config
+        Sequel::SchemaSharding.config
+      end
+    end
+  end
+end

data/lib/sequel/schema-sharding/database_manager.rb

@@ -0,0 +1,105 @@
+require 'sequel'
+require 'sequel/schema-sharding/connection_manager'
+
+Sequel.extension :migration
+
+module Sequel
+  module SchemaSharding
+    class DatabaseManager
+      def create_databases
+        config.physical_shard_configs.each_pair do |name, config|
+          begin
+            # Need to create connection manually with specifying a database in order to create the database
+            connection = Sequel.postgres(:user => config['username'],
+              :password => config['password'],
+              :host => config['host'])
+
+            Sequel::SchemaSharding.logger.info "Creating #{config['database']}.."
+
+            connection.run("CREATE DATABASE #{config['database']}")
+          rescue Sequel::DatabaseError => e
+            if e.message.include?('already exists')
+              $stderr.puts "#{config['database']} database already exists"
+            else
+              raise e
+            end
+          ensure
+            connection.disconnect
+          end
+        end
+      end
+
+      def drop_databases
+        connection_manager.disconnect
+        config.physical_shard_configs.each_pair do |name, config|
+          # Need to create connection manually with specifying a database in order to create the database
+          begin
+            connection = Sequel.postgres(:user => config['username'],
+              :password => config['password'],
+              :host => config['host'])
+
+            Sequel::SchemaSharding.logger.info "Dropping #{config['database']}.."
+            connection.run("DROP DATABASE #{config['database']}")
+          rescue Sequel::DatabaseError => e
+            if e.message.include?('does not exist')
+              $stderr.puts "#{config['database']} database doesnt exist"
+            else
+              raise e
+            end
+          ensure
+            connection.disconnect
+          end
+        end
+      end
+
+      def create_shards
+        config.table_names.each do |table_name|
+          config.logical_shard_configs(table_name).each_pair do |shard_number, physical_shard|
+            schema_name = connection_manager.schema_for(table_name, env, shard_number)
+            Sequel::SchemaSharding.logger.info "Creating schema #{schema_name} on #{physical_shard}.."
+            connection = connection_manager[physical_shard]
+
+            begin
+              connection.run("CREATE SCHEMA #{schema_name}")
+            rescue Sequel::DatabaseError => e
+              if e.message.include?('already exists')
+                $stderr.puts "#{schema_name} schema already exists"
+              else
+                raise e
+              end
+            end
+
+            connection.run("SET search_path TO #{schema_name}")
+
+            Sequel::Migrator.run(connection, Sequel::SchemaSharding.migration_path + "/#{table_name}", :use_transactions => true)
+          end
+        end
+      end
+
+      def drop_shards
+        config.table_names.each do |table_name|
+          config.logical_shard_configs(table_name).each_pair do |shard_number, physical_shard|
+            schema_name = connection_manager.schema_for(table_name, env, shard_number)
+            Sequel::SchemaSharding.logger.info "Dropping schema #{schema_name} on #{physical_shard}.."
+            connection = connection_manager[physical_shard]
+            connection.run("DROP SCHEMA #{schema_name} CASCADE")
+          end
+        end
+      end
+
+      private
+
+      def env
+        config.env
+      end
+
+      def config
+        Sequel::SchemaSharding.config
+      end
+
+      def connection_manager
+        Sequel::SchemaSharding.connection_manager
+      end
+    end
+  end
+end

data/lib/sequel/schema-sharding/finder.rb

@@ -0,0 +1,43 @@
+require 'singleton'
+
+module Sequel
+  module SchemaSharding
+    class Finder
+      class Result
+        attr_reader :connection, :schema
+
+        def initialize(connection, schema)
+          @connection = connection
+          @schema = schema
+        end
+      end
+
+      include ::Singleton
+
+      def lookup(table_name, id)
+        shard_number = shard_for_id(table_name, id)
+        physical_shard = config.logical_shard_configs(table_name)[shard_number]
+
+        conn = Sequel::SchemaSharding.connection_manager[physical_shard]
+        schema = Sequel::SchemaSharding.connection_manager.schema_for(table_name, config.env, shard_number)
+
+        Result.new(conn, schema)
+      end
+
+      private
+
+      def shard_for_id(table_name, id)
+        ring(table_name).shard_for_id(id)
+      end
+
+      def ring(table_name)
+        @rings ||= {}
+        @rings[table_name] ||= Sequel::SchemaSharding::Ring.new(config.logical_shard_configs(table_name).keys)
+      end
+
+      def config
+        Sequel::SchemaSharding.config
+      end
+    end
+  end
+end

data/lib/sequel/schema-sharding/model.rb

@@ -0,0 +1,91 @@
+require 'sequel'
+
+module Sequel
+  module SchemaSharding
+    # Extensions to the Sequel model to allow logical/physical shards. Actual table models should
+    # inherit this class like so:
+    #
+    # class Cat < Sequel::SchemaSharding::Model
+    #   set_columns [:cat_id, :fur, :tongue, :whiskers] # Columns in the database need to be predefined.
+    #   set_sharded_column :cat_id # Define the shard column
+    #
+    #   def self.by_cat_id(id)
+    #     # You should always call shard_for in finders to select the correct connection.
+    #     shard_for(id).where(cat_id: id)
+    #   end
+    # end
+
+    def self.Model(source)
+      klass = Sequel::Model(Sequel::SchemaSharding.connection_manager.default_dataset_for(source))
+
+      klass.include(SchemaSharding::ShardedModel)
+
+      klass
+    end
+
+    module ShardedModel
+
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+
+        #protected
+
+        # Set the column on which the current model is sharded. This is used when saving, inserting and finding
+        # to decide which connection to use.
+        def set_sharded_column(column)
+          @sharded_column = column
+        end
+
+        # Accessor for the sharded_columns
+        def sharded_column
+          @sharded_column
+        end
+
+        # Return a valid Sequel::Dataset that is tied to the shard table and connection for the id and will load values
+        # run by the query into the model.
+        def shard_for(id)
+          result = self.result_for(id)
+          ds = result.connection[schema_and_table(result)]
+          ds.row_proc = self
+          dataset_method_modules.each { |m| ds.instance_eval { extend(m) } }
+          ds.model = self
+          ds
+        end
+
+        # The result of a lookup for the given id. See Sequel::SchemaSharding::Finder::Result
+        def result_for(id)
+          Sequel::SchemaSharding::Finder.instance.lookup(self.implicit_table_name, id)
+        end
+
+        # Construct the schema and table for use in a dataset.
+        def schema_and_table(result)
+          :"#{result.schema}__#{self.implicit_table_name}"
+        end
+      end
+
+      # The database connection that has the logical shard.
+      def db
+        @db ||= finder_result.connection
+      end
+
+      # Wrapper for performing the sharding lookup based on the sharded column.
+      def finder_result
+        @result ||= self.class.result_for(self.send(self.class.sharded_column))
+      end
+
+      # Dataset instance based on the sharded column.
+      def this_server
+        @this_server ||= db[self.class.schema_and_table(finder_result)]
+      end
+
+      # Overriden to not use @dataset value from the Sequel::Model. Used internally only.
+      def _insert_dataset
+        this_server
+      end
+
+    end
+  end
+end

data/lib/sequel/schema-sharding/ring.rb

@@ -0,0 +1,118 @@
+require 'digest/sha1'
+require 'zlib'
+
+module Sequel
+  module SchemaSharding
+    class Ring
+      POINTS_PER_SERVER = 1
+
+      attr_accessor :shards, :continuum
+
+      def initialize(shards)
+        @shards = shards
+        @continuum = nil
+        if shards.size > 1
+          continuum = []
+          shards.each do |shard|
+            hash = Digest::SHA1.hexdigest("#{shard}")
+            value = Integer("0x#{hash[0..7]}")
+            continuum << Entry.new(value, shard)
+          end
+          @continuum = continuum.sort { |a, b| a.value <=> b.value }
+        end
+      end
+
+      def shard_for_id(id)
+        if @continuum
+          hkey = hash_for(id)
+          entryidx = binary_search(@continuum, hkey)
+          return @continuum[entryidx].server
+        else
+          server = @servers.first
+          return server if server
+        end
+
+        raise StandardError, "No server available"
+      end
+
+      private
+
+      def hash_for(key)
+        Zlib.crc32(key.to_s)
+      end
+
+      def entry_count
+        ((shards.size * POINTS_PER_SERVER)).floor
+      end
+
+      # Native extension to perform the binary search within the continuum
+      # space.  Fallback to a pure Ruby version if the compilation doesn't work.
+      # optional for performance and only necessary if you are using multiple
+      # memcached servers.
+      begin
+        require 'inline'
+        inline do |builder|
+          builder.c <<-EOM
+              int binary_search(VALUE ary, unsigned int r) {
+                  long upper = RARRAY_LEN(ary) - 1;
+                  long lower = 0;
+                  long idx = 0;
+                  ID value = rb_intern("value");
+                  VALUE continuumValue;
+                  unsigned int l;
+
+                  while (lower <= upper) {
+                      idx = (lower + upper) / 2;
+
+                      continuumValue = rb_funcall(RARRAY_PTR(ary)[idx], value, 0);
+                      l = NUM2UINT(continuumValue);
+                      if (l == r) {
+                          return idx;
+                      }
+                      else if (l > r) {
+                          upper = idx - 1;
+                      }
+                      else {
+                          lower = idx + 1;
+                      }
+                  }
+                  return upper;
+              }
+          EOM
+        end
+      rescue LoadError
+        # Find the closest index in the Ring with value <= the given value
+        def binary_search(ary, value)
+          upper = ary.size - 1
+          lower = 0
+          idx = 0
+
+          while (lower <= upper) do
+            idx = (lower + upper) / 2
+            comp = ary[idx].value <=> value
+
+            if comp == 0
+              return idx
+            elsif comp > 0
+              upper = idx - 1
+            else
+              lower = idx + 1
+            end
+          end
+          return upper
+        end
+      end
+
+      class Entry
+        attr_reader :value
+        attr_reader :server
+
+        def initialize(val, srv)
+          @value = val
+          @server = srv
+        end
+      end
+
+    end
+  end
+end

data/lib/sequel/schema-sharding/sequel_ext.rb

@@ -0,0 +1,14 @@
+module Sequel
+  module SchemaSharding
+    module SequelExt
+      module ClassMethods
+        def db
+          return @db if @db
+        end
+      end
+    end
+  end
+end
+
+Sequel::Model.plugin Sequel::SchemaSharding::SequelExt
+Sequel::Model.plugin :validation_helpers

data/lib/sequel/schema-sharding/version.rb

@@ -0,0 +1,5 @@
+module Sequel
+  module SchemaSharding
+    VERSION = "0.0.1"
+  end
+end

data/lib/sequel/tasks/test.rake

@@ -0,0 +1,24 @@
+require 'sequel/schema-sharding'
+
+namespace :sequel do
+  namespace :db do
+    desc 'Create databases and shards for tests'
+    task :create do
+      ENV['RACK_ENV'] ||= 'test'
+      Sequel::SchemaSharding.sharding_yml_path = "spec/fixtures/test_db_config.yml"
+      Sequel::SchemaSharding.migration_path = "spec/fixtures/db/migrate"
+      manager = Sequel::SchemaSharding::DatabaseManager.new
+      manager.create_databases
+      manager.create_shards
+    end
+
+    desc 'Create databases and shards for tests'
+    task :drop do
+      ENV['RACK_ENV'] ||= 'test'
+      Sequel::SchemaSharding.sharding_yml_path = "spec/fixtures/test_db_config.yml"
+      Sequel::SchemaSharding.migration_path = "spec/fixtures/db/migrate"
+      manager = Sequel::SchemaSharding::DatabaseManager.new
+      manager.drop_databases
+    end
+  end
+end

data/sequel-schema-sharding.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sequel/schema-sharding/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "sequel-schema-sharding"
+  spec.version       = Sequel::SchemaSharding::VERSION
+  spec.authors       = ["Paul Henry", "James Hart", "Eric Saxby"]
+  spec.email         = ["dev@wanelo.com"]
+  spec.description   = %q{}
+  spec.summary       = %q{Create horizontally sharded Sequel models with Postgres}
+  spec.homepage      = "https://github.com/wanelo/sequel-schema-sharding"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "sequel"
+  spec.add_dependency "pg"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/spec/fixtures/db/migrate/artists/001_create_test_table.rb

@@ -0,0 +1,13 @@
+Sequel.migration do
+  up do
+    create_table(:artists) do
+      primary_key :id
+      Integer :artist_id
+      String :name, :null=>false
+    end
+  end
+
+  down do
+    drop_table(:artists)
+  end
+end

data/spec/fixtures/db/migrate/boof/001_create_boof_table.rb

@@ -0,0 +1,12 @@
+Sequel.migration do
+  up do
+    create_table(:boofs) do
+      primary_key :id
+      String :name, :null=>false
+    end
+  end
+
+  down do
+    drop_table(:boofs)
+  end
+end

data/spec/fixtures/test_db_config.yml

@@ -0,0 +1,67 @@
+test:
+  tables:
+    artists:
+      schema_name: sequel_logical_artists_%e_%s
+      logical_shards:
+        1..10: shard1
+        11..20: shard2
+    boof:
+      schema_name: sequel_logical_boof_%e_%s
+      logical_shards:
+        1..10: shard1
+        11..20: shard2
+  physical_shards:
+    shard1:
+      host: 127.0.0.1
+      database: sequel_test_shard1
+    shard2:
+      host: 127.0.0.1
+      database: sequel_test_shard2
+  common:
+    username: postgres
+    password: boomboomkaboom
+boom:
+  tables:
+    artists:
+      schema_name: sequel_explosions_artists_%e_%s
+      logical_shards:
+        1..10: shard1
+        11..20: shard2
+    boof:
+      schema_name: sequel_explosions_boof_%e_%s
+      logical_shards:
+        1..10: shard1
+        11..20: shard2
+  physical_shards:
+    shard1:
+      host: 127.0.0.1
+      database: sequel_boom_shard1
+    shard2:
+      host: 127.0.0.1
+      database: sequel_boom_shard2
+  common:
+    username: postgres
+    password: boomboomkaboom
+    port: 5432
+#boom:
+#  logical_shards:
+#    user_saves:
+#      1..10: shard1
+#      11..20: shard2
+#    product_saves:
+#      1..10: shard2
+#      11..20: shard3
+#  physical_shards:
+#    shard1:
+#      host: 127.0.0.1
+#      database: wanelo_saves_boom_shard1
+#    shard2:
+#      host: 127.0.0.1
+#      database: wanelo_saves_boom_shard2
+#    shard3:
+#      host: 127.0.0.1
+#      database: wanelo_saves_boom_shard3
+#  common:
+#    username: postgres
+#    password: boomboomkaboom
+#    port: 5432

data/spec/schema-sharding/configuration_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+require 'sequel/schema-sharding/configuration'
+
+describe Sequel::SchemaSharding::Configuration do
+
+  let(:config) { Sequel::SchemaSharding::Configuration.new(:boom, 'spec/fixtures/test_db_config.yml') }
+
+  describe '#logical_shard_configs' do
+    it 'returns a hash representing the mapping between logical and physical shards' do
+      shards = config.logical_shard_configs('boof')
+
+      expect(shards.length).to eq(20)
+
+      shards.each_pair do |key, value|
+        if key <= 10
+          expect(value).to eq('shard1')
+        elsif key > 10
+          expect(value).to eq('shard2')
+        end
+      end
+    end
+  end
+
+  describe '#physical_shard_configs' do
+    it 'returns a hash representing the configuration for all physical shards' do
+      shards = config.physical_shard_configs
+
+      expect(shards.length).to eq(2)
+
+      i = 1
+      shards.each_pair do |key, value|
+        expect(value['host']).to eq('127.0.0.1')
+        expect(value['database']).to eq("sequel_boom_shard#{i}")
+        expect(value['username']).to eq('postgres')
+        expect(value['password']).to eq('boomboomkaboom')
+        expect(value['port']).to eq(5432)
+
+        i += 1
+      end
+    end
+  end
+end

data/spec/schema-sharding/connection_manager_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+require 'sequel/schema-sharding/connection_manager'
+
+describe Sequel::SchemaSharding::ConnectionManager do
+  let(:config) { Sequel::SchemaSharding::Configuration.new('boom', 'spec/fixtures/test_db_config.yml') }
+
+  subject { Sequel::SchemaSharding::ConnectionManager.new }
+
+  before { subject.stubs(:config).returns(config) }
+
+  describe '#[]' do
+    it 'returns a valid connection instance for the specified physical shard' do
+      expect(subject['shard1']).to be_a(Sequel::Postgres::Database)
+      expect(subject['shard2']).to be_a(Sequel::Postgres::Database)
+    end
+  end
+
+  describe "#schema_for" do
+    it "returns the schema name based on env and shard number" do
+      subject.schema_for('boof', 'pickles', 3).should eq 'sequel_explosions_boof_pickles_3'
+    end
+  end
+
+  describe "#default_dataset_for" do
+    it "returns a dataset scoped to a configured schema" do
+      # TODO ConnectionManager is dependent on global state from Sequel::SchemaSharding.config.
+      #      This should be deconstructed to allow for injection of a mock config for testing.
+      dataset = subject.default_dataset_for("artists")
+      expect(dataset).to be_a(Sequel::Dataset)
+      expect(dataset.first_source_table).to eql(:'sequel_explosions_artists_test_1__artists')
+    end
+  end
+end

data/spec/schema-sharding/database_manager_spec.rb

@@ -0,0 +1,124 @@
+require 'spec_helper'
+require 'sequel/schema-sharding'
+
+describe Sequel::SchemaSharding::DatabaseManager, type: :manager, sharded: true do
+
+  let(:config) { Sequel::SchemaSharding::Configuration.new('boom', 'spec/fixtures/test_db_config.yml') }
+
+  after do
+    Sequel::SchemaSharding.connection_manager.disconnect
+  end
+
+  around do |ex|
+    Sequel::SchemaSharding.stubs(:config).returns(config)
+
+    @manager = Sequel::SchemaSharding::DatabaseManager.new
+    @manager.send(:connection_manager).disconnect
+    DatabaseHelper.disconnect
+
+    DatabaseHelper.drop_db('sequel_boom_shard1')
+    DatabaseHelper.drop_db('sequel_boom_shard2')
+
+    ex.call
+  end
+
+  describe '#create_database' do
+    context 'database does not exist' do
+      it 'creates the database for the current environment' do
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
+        @manager.create_databases
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
+      end
+    end
+
+    context 'database exists' do
+
+      before(:each) do
+        @manager.create_databases
+      end
+
+      it 'outputs message to stderr' do
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
+        $stderr.expects(:puts).with(regexp_matches(/already exists/)).twice
+        @manager.create_databases
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
+      end
+    end
+  end
+
+  describe '#drop_databases' do
+    context 'databases exist' do
+      it 'drops the database for the current environment' do
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
+
+        @manager.create_databases
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
+
+        @manager.drop_databases
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
+      end
+    end
+
+    context 'databases dont exist' do
+      it 'raises an error' do
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
+        expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
+
+        $stderr.expects(:puts).with(regexp_matches(/database doesnt exist/)).times(2)
+
+        @manager.drop_databases
+      end
+    end
+  end
+
+  describe 'logical shards' do
+    before(:each) do
+      @manager.create_databases
+    end
+
+    describe '#create_shards' do
+      it 'creates the database structure' do
+        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
+        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
+        @manager.create_shards
+        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
+        expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
+      end
+
+      context 'shards already exist' do
+        it 'prints that shards already exist' do
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
+          @manager.create_shards
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
+
+          $stderr.expects(:puts).with(regexp_matches(/already exists/)).at_least_once
+          @manager.create_shards
+        end
+      end
+    end
+
+    describe '#drop_schemas' do
+      context 'schemas exist' do
+        it 'drops the schemas' do
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
+          @manager.create_shards
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
+          @manager.drop_shards
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
+          expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
+        end
+      end
+    end
+  end
+end

data/spec/schema-sharding/finder_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+require 'sequel/schema-sharding'
+
+describe Sequel::SchemaSharding::Finder do
+
+  describe '.lookup' do
+    it 'returns an object with a valid connection and schema' do
+      result = Sequel::SchemaSharding::Finder.instance.lookup('boof', 60)
+      expect(result.connection).to be_a(Sequel::Postgres::Database)
+      expect(result.schema).to eq('sequel_logical_boof_test_2')
+    end
+  end
+end

data/spec/schema-sharding/model_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe Sequel::SchemaSharding, 'Model' do
+
+  let(:model) do
+    klass = Sequel::SchemaSharding::Model('artists')
+    klass.instance_eval do
+      set_sharded_column :artist_id
+      set_columns [:artist_id, :name]
+
+      def by_id(id)
+        shard_for(id).where(artist_id: id)
+      end
+
+      def self.implicit_table_name
+        'artists'
+      end
+    end
+    klass
+  end
+
+  describe '#by_id' do
+    it 'returns a valid artist by id' do
+      artist = model.create(artist_id: 14, name: 'Paul')
+      expect(artist.id).to_not be_nil
+      read_back_artist = model.by_id(14).first
+      expect(read_back_artist).to be_a(model)
+      expect(read_back_artist.name).to eql('Paul')
+    end
+  end
+
+  describe '#create' do
+    it 'creates a valid artist' do
+      artist = model.create(artist_id: 234, name: 'Paul')
+      expect(artist).to be_a(model)
+      expect(artist.name).to eql('Paul')
+    end
+  end
+
+end

data/spec/schema-sharding/ring_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+require 'sequel/schema-sharding'
+
+describe Sequel::SchemaSharding::Ring do
+
+  describe 'a ring of servers' do
+    it 'have the continuum sorted by value' do
+      shards = [1, 2, 3, 4, 5, 6, 7, 8]
+      ring = Sequel::SchemaSharding::Ring.new(shards)
+      previous_value = 0
+      ring.continuum.each do |entry|
+        expect(entry.value).to be > previous_value
+        previous_value = entry.value
+      end
+    end
+  end
+
+  describe '#shard_for_id' do
+    it 'returns a server for a given id' do
+      shards = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
+      ring = Sequel::SchemaSharding::Ring.new(shards)
+      expect(ring.shard_for_id(3489409)).to eq(4)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,53 @@
+ENV['RACK_ENV'] ||= 'test'
+
+require 'bundler/setup'
+Bundler.require 'test'
+
+require 'sequel-schema-sharding'
+require 'support/database_helper'
+require 'mocha/api'
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+  config.alias_example_to :fit, focus: true
+  config.mock_framework = :mocha
+
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+
+  config.before :all do |ex|
+    Sequel::SchemaSharding.logger = Logger.new(StringIO.new)
+    Sequel::SchemaSharding.sharding_yml_path = "spec/fixtures/test_db_config.yml"
+    Sequel::SchemaSharding.migration_path = "spec/fixtures/db/migrate"
+  end
+
+  config.around :each do |ex|
+    #Sequel::SchemaSharding.config = Sequel::SchemaSharding::Configuration.new('boom', 'spec/fixtures/test_db_config.yml')
+
+    # Start transactions in each connection to the physical shards
+    connections = Sequel::SchemaSharding.config.physical_shard_configs.map do |shard_config|
+      Sequel::SchemaSharding.connection_manager[shard_config[0]]
+    end
+
+    start_transaction_proc = Proc.new do |connections|
+      if connections.length == 0
+        ex.run
+      else
+        connections[0].transaction do
+          connections.shift
+          start_transaction_proc.call(connections)
+          raise Sequel::Rollback
+        end
+      end
+    end
+
+    start_transaction_proc.call(connections)
+  end
+
+end

data/spec/support/database_helper.rb

@@ -0,0 +1,35 @@
+require 'sequel'
+
+class DatabaseHelper
+  def self.db
+    @db = Sequel.postgres(user: 'postgres', password: 'alkfjasdlfj', host: 'localhost')
+  end
+
+  def self.disconnect
+    db.disconnect
+  end
+
+  def self.drop_db(database)
+    if db_exists?("#{database}")
+      self.disconnect
+
+      db.run("DROP DATABASE #{database}")
+    end
+  end
+
+  def self.db_exists?(database)
+    dbs.include?(database)
+  end
+
+  def self.dbs
+    db.fetch('SELECT datname FROM pg_database WHERE datistemplate = false;').all.map { |d| d[:datname] }
+  end
+
+  def self.schema_exists?(database, schema)
+    schemas(database).include?(schema)
+  end
+
+  def self.schemas(database)
+    Sequel::SchemaSharding.connection_manager[database].fetch('select nspname from pg_catalog.pg_namespace;').all.map { |d| d[:nspname] }
+  end
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: sequel-schema-sharding
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Paul Henry
+- James Hart
+- Eric Saxby
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- dev@wanelo.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/sequel-schema-sharding.rb
+- lib/sequel/schema-sharding.rb
+- lib/sequel/schema-sharding/configuration.rb
+- lib/sequel/schema-sharding/connection_manager.rb
+- lib/sequel/schema-sharding/database_manager.rb
+- lib/sequel/schema-sharding/finder.rb
+- lib/sequel/schema-sharding/model.rb
+- lib/sequel/schema-sharding/ring.rb
+- lib/sequel/schema-sharding/sequel_ext.rb
+- lib/sequel/schema-sharding/version.rb
+- lib/sequel/tasks/test.rake
+- sequel-schema-sharding.gemspec
+- spec/fixtures/db/migrate/artists/001_create_test_table.rb
+- spec/fixtures/db/migrate/boof/001_create_boof_table.rb
+- spec/fixtures/test_db_config.yml
+- spec/schema-sharding/configuration_spec.rb
+- spec/schema-sharding/connection_manager_spec.rb
+- spec/schema-sharding/database_manager_spec.rb
+- spec/schema-sharding/finder_spec.rb
+- spec/schema-sharding/model_spec.rb
+- spec/schema-sharding/ring_spec.rb
+- spec/spec_helper.rb
+- spec/support/database_helper.rb
+homepage: https://github.com/wanelo/sequel-schema-sharding
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.7
+signing_key: 
+specification_version: 4
+summary: Create horizontally sharded Sequel models with Postgres
+test_files:
+- spec/fixtures/db/migrate/artists/001_create_test_table.rb
+- spec/fixtures/db/migrate/boof/001_create_boof_table.rb
+- spec/fixtures/test_db_config.yml
+- spec/schema-sharding/configuration_spec.rb
+- spec/schema-sharding/connection_manager_spec.rb
+- spec/schema-sharding/database_manager_spec.rb
+- spec/schema-sharding/finder_spec.rb
+- spec/schema-sharding/model_spec.rb
+- spec/schema-sharding/ring_spec.rb
+- spec/spec_helper.rb
+- spec/support/database_helper.rb