sequel-schema-sharding 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 9a83484015fdcd8dc4799e752d6d82ee94498738
4
+ data.tar.gz: 2e44e0d65a6c598371035daddc1724022b3c893b
5
+ SHA512:
6
+ metadata.gz: bac9d7a0ed39dbe558e2f0bd6422bda34f68676662ba5283722340c1ffbf784f6e8c433ca6b118efe41c8a0e66ed123c8bfd2f482597ace1ab750a2b7d2fcbd1
7
+ data.tar.gz: 724cae47d487ee6e63a8e969ae8ed9f1d380ffaf61b3877074578a9704565fd0290ffd27550904afc9ad3949aba536b4fa69d5209670a39b13f21ae2e0fd874c
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --color
2
+ --format progress
data/.travis.yml ADDED
@@ -0,0 +1,8 @@
1
+ language: ruby
2
+ rvm:
3
+ - 1.9.3
4
+ - 2.0.0-p247
5
+ before_script: "bundle exec rake sequel:db:create"
6
+ script: "bundle exec rspec"
7
+ notifications:
8
+ email: false
data/Gemfile ADDED
@@ -0,0 +1,10 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in sequel-sharding.gemspec
4
+ gemspec
5
+
6
+ group :test do
7
+ gem 'rspec'
8
+ gem 'mocha', require: false
9
+ gem 'pry-nav'
10
+ end
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 TODO: Write your name
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,33 @@
1
+ Sequel::SchemaSharding
2
+ ================
3
+
4
+ [![Build Status](https://travis-ci.org/wanelo/sequel-sharding.png?branch=master)](https://travis-ci.org/wanelo/sequel-sharding)
5
+
6
+ Horizontally shard postgres with the Sequel gem. This gem allows you to configure mappings between logical and
7
+ physical shards, and pool connections between logical shards on the same physical server.
8
+
9
+ ## Installation
10
+
11
+ Add this line to your application's Gemfile:
12
+
13
+ gem 'sequel-sharding'
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install sequel-sharding
22
+
23
+ ## Usage
24
+
25
+ TODO :)
26
+
27
+ ## Contributing
28
+
29
+ 1. Fork it
30
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
31
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
32
+ 4. Push to the branch (`git push origin my-new-feature`)
33
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require "bundler/gem_tasks"
2
+ import 'lib/sequel/tasks/test.rake'
@@ -0,0 +1 @@
1
+ require 'sequel/schema-sharding'
@@ -0,0 +1,54 @@
1
+ require 'sequel/schema-sharding/version'
2
+ require 'sequel/schema-sharding/configuration'
3
+ require 'sequel/schema-sharding/connection_manager'
4
+ require 'sequel/schema-sharding/database_manager'
5
+ require 'sequel/schema-sharding/ring'
6
+ require 'sequel/schema-sharding/finder'
7
+ require 'sequel/schema-sharding/sequel_ext'
8
+ require 'sequel/schema-sharding/model'
9
+ require 'logger'
10
+
11
+ module Sequel
12
+ module SchemaSharding
13
+ def self.config
14
+ @config ||= Sequel::SchemaSharding::Configuration.new(ENV['RACK_ENV'], sharding_yml_path)
15
+ end
16
+
17
+ def self.config=(config)
18
+ @config = config
19
+ end
20
+
21
+ def self.logger
22
+ @logger ||= Logger.new($stdout)
23
+ end
24
+
25
+ def self.logger=(logger)
26
+ @logger = logger
27
+ end
28
+
29
+ def self.connection_manager
30
+ @connection_manager ||= ConnectionManager.new
31
+ end
32
+
33
+ def self.connection_manager=(connection_manager)
34
+ @connection_manager = connection_manager
35
+ end
36
+
37
+ def self.sharding_yml_path
38
+ @sharding_yml_path ||= File.expand_path('../../../config/sharding.yml', __FILE__)
39
+ end
40
+
41
+ def self.sharding_yml_path=(path)
42
+ @sharding_yml_path = path
43
+ end
44
+
45
+ def self.migration_path
46
+ @migration_path || raise('You must set the migration path.')
47
+ end
48
+
49
+ def self.migration_path=(path)
50
+ @migration_path = path
51
+ end
52
+
53
+ end
54
+ end
@@ -0,0 +1,54 @@
1
+ require 'yaml'
2
+
3
+ module Sequel
4
+ module SchemaSharding
5
+ class Configuration
6
+ attr_reader :env, :yaml_path
7
+
8
+ def initialize(env, yaml_path)
9
+ @env = env
10
+ @yaml_path = yaml_path
11
+ end
12
+
13
+ def physical_shard_configs
14
+ @physical_shard_configs ||= config['physical_shards'].inject({}) do |hash, value|
15
+ hash[value[0]] = config['common'].merge(value[1])
16
+ hash
17
+ end
18
+ end
19
+
20
+ def logical_shard_configs(table_name)
21
+ table_name = table_name.to_s
22
+ @logical_shard_table_configs ||= {}
23
+ @logical_shard_table_configs[table_name] ||= begin
24
+ table_configs = config['tables'][table_name]
25
+ raise "Unknown table #{table_name} in configuration" if table_configs.nil?
26
+ table_configs['logical_shards'].inject({}) do |hash, value|
27
+ eval(value[0]).each do |i|
28
+ hash[i] = value[1]
29
+ end
30
+ hash
31
+ end
32
+ end
33
+ end
34
+
35
+ def table_names
36
+ config['tables'].keys
37
+ end
38
+
39
+ def schema_name(table_name)
40
+ config['tables'][table_name.to_s]['schema_name']
41
+ end
42
+
43
+ private
44
+
45
+ def config
46
+ yaml[env.to_s]
47
+ end
48
+
49
+ def yaml
50
+ @raw_yaml ||= YAML.load_file(yaml_path)
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,48 @@
1
+ require 'singleton'
2
+
3
+ module Sequel
4
+ module SchemaSharding
5
+ class ConnectionManager
6
+ attr_reader :connections
7
+
8
+ def initialize
9
+ @connections = {}
10
+ end
11
+
12
+ def [](name)
13
+ config = db_config_for(name)
14
+ @connections[name.to_s] ||= Sequel.postgres(:user => config['username'],
15
+ :password => config['password'],
16
+ :host => config['host'],
17
+ :database => config['database'])
18
+ end
19
+
20
+ def disconnect
21
+ @connections.each_value do |conn|
22
+ conn.disconnect
23
+ end
24
+ @connections = {}
25
+ end
26
+
27
+ def schema_for(table_name, environment, shard_number)
28
+ config.schema_name(table_name).gsub('%e', environment).gsub('%s', shard_number.to_s)
29
+ end
30
+
31
+ def default_dataset_for(table_name)
32
+ shard_number = config.logical_shard_configs(table_name).keys.first
33
+ shard_name = config.logical_shard_configs(table_name)[shard_number]
34
+ self[shard_name][:"#{schema_for(table_name, ENV['RACK_ENV'], shard_number)}__#{table_name}"]
35
+ end
36
+
37
+ private
38
+
39
+ def db_config_for(name)
40
+ config.physical_shard_configs[name]
41
+ end
42
+
43
+ def config
44
+ Sequel::SchemaSharding.config
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,105 @@
1
+ require 'sequel'
2
+ require 'sequel/schema-sharding/connection_manager'
3
+
4
+ Sequel.extension :migration
5
+
6
+ module Sequel
7
+ module SchemaSharding
8
+ class DatabaseManager
9
+ def create_databases
10
+ config.physical_shard_configs.each_pair do |name, config|
11
+ begin
12
+ # Need to create connection manually with specifying a database in order to create the database
13
+ connection = Sequel.postgres(:user => config['username'],
14
+ :password => config['password'],
15
+ :host => config['host'])
16
+
17
+ Sequel::SchemaSharding.logger.info "Creating #{config['database']}.."
18
+
19
+ connection.run("CREATE DATABASE #{config['database']}")
20
+ rescue Sequel::DatabaseError => e
21
+ if e.message.include?('already exists')
22
+ $stderr.puts "#{config['database']} database already exists"
23
+ else
24
+ raise e
25
+ end
26
+ ensure
27
+ connection.disconnect
28
+ end
29
+ end
30
+ end
31
+
32
+ def drop_databases
33
+ connection_manager.disconnect
34
+ config.physical_shard_configs.each_pair do |name, config|
35
+ # Need to create connection manually with specifying a database in order to create the database
36
+ begin
37
+ connection = Sequel.postgres(:user => config['username'],
38
+ :password => config['password'],
39
+ :host => config['host'])
40
+
41
+ Sequel::SchemaSharding.logger.info "Dropping #{config['database']}.."
42
+ connection.run("DROP DATABASE #{config['database']}")
43
+ rescue Sequel::DatabaseError => e
44
+ if e.message.include?('does not exist')
45
+ $stderr.puts "#{config['database']} database doesnt exist"
46
+ else
47
+ raise e
48
+ end
49
+ ensure
50
+ connection.disconnect
51
+ end
52
+ end
53
+ end
54
+
55
+ def create_shards
56
+ config.table_names.each do |table_name|
57
+ config.logical_shard_configs(table_name).each_pair do |shard_number, physical_shard|
58
+ schema_name = connection_manager.schema_for(table_name, env, shard_number)
59
+ Sequel::SchemaSharding.logger.info "Creating schema #{schema_name} on #{physical_shard}.."
60
+ connection = connection_manager[physical_shard]
61
+
62
+ begin
63
+ connection.run("CREATE SCHEMA #{schema_name}")
64
+ rescue Sequel::DatabaseError => e
65
+ if e.message.include?('already exists')
66
+ $stderr.puts "#{schema_name} schema already exists"
67
+ else
68
+ raise e
69
+ end
70
+ end
71
+
72
+ connection.run("SET search_path TO #{schema_name}")
73
+
74
+ Sequel::Migrator.run(connection, Sequel::SchemaSharding.migration_path + "/#{table_name}", :use_transactions => true)
75
+ end
76
+ end
77
+ end
78
+
79
+ def drop_shards
80
+ config.table_names.each do |table_name|
81
+ config.logical_shard_configs(table_name).each_pair do |shard_number, physical_shard|
82
+ schema_name = connection_manager.schema_for(table_name, env, shard_number)
83
+ Sequel::SchemaSharding.logger.info "Dropping schema #{schema_name} on #{physical_shard}.."
84
+ connection = connection_manager[physical_shard]
85
+ connection.run("DROP SCHEMA #{schema_name} CASCADE")
86
+ end
87
+ end
88
+ end
89
+
90
+ private
91
+
92
+ def env
93
+ config.env
94
+ end
95
+
96
+ def config
97
+ Sequel::SchemaSharding.config
98
+ end
99
+
100
+ def connection_manager
101
+ Sequel::SchemaSharding.connection_manager
102
+ end
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,43 @@
1
+ require 'singleton'
2
+
3
+ module Sequel
4
+ module SchemaSharding
5
+ class Finder
6
+ class Result
7
+ attr_reader :connection, :schema
8
+
9
+ def initialize(connection, schema)
10
+ @connection = connection
11
+ @schema = schema
12
+ end
13
+ end
14
+
15
+ include ::Singleton
16
+
17
+ def lookup(table_name, id)
18
+ shard_number = shard_for_id(table_name, id)
19
+ physical_shard = config.logical_shard_configs(table_name)[shard_number]
20
+
21
+ conn = Sequel::SchemaSharding.connection_manager[physical_shard]
22
+ schema = Sequel::SchemaSharding.connection_manager.schema_for(table_name, config.env, shard_number)
23
+
24
+ Result.new(conn, schema)
25
+ end
26
+
27
+ private
28
+
29
+ def shard_for_id(table_name, id)
30
+ ring(table_name).shard_for_id(id)
31
+ end
32
+
33
+ def ring(table_name)
34
+ @rings ||= {}
35
+ @rings[table_name] ||= Sequel::SchemaSharding::Ring.new(config.logical_shard_configs(table_name).keys)
36
+ end
37
+
38
+ def config
39
+ Sequel::SchemaSharding.config
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,91 @@
1
+ require 'sequel'
2
+
3
+ module Sequel
4
+ module SchemaSharding
5
+ # Extensions to the Sequel model to allow logical/physical shards. Actual table models should
6
+ # inherit this class like so:
7
+ #
8
+ # class Cat < Sequel::SchemaSharding::Model
9
+ # set_columns [:cat_id, :fur, :tongue, :whiskers] # Columns in the database need to be predefined.
10
+ # set_sharded_column :cat_id # Define the shard column
11
+ #
12
+ # def self.by_cat_id(id)
13
+ # # You should always call shard_for in finders to select the correct connection.
14
+ # shard_for(id).where(cat_id: id)
15
+ # end
16
+ # end
17
+
18
+ def self.Model(source)
19
+ klass = Sequel::Model(Sequel::SchemaSharding.connection_manager.default_dataset_for(source))
20
+
21
+ klass.include(SchemaSharding::ShardedModel)
22
+
23
+ klass
24
+ end
25
+
26
+ module ShardedModel
27
+
28
+ def self.included(base)
29
+ base.extend(ClassMethods)
30
+ end
31
+
32
+ module ClassMethods
33
+
34
+ #protected
35
+
36
+ # Set the column on which the current model is sharded. This is used when saving, inserting and finding
37
+ # to decide which connection to use.
38
+ def set_sharded_column(column)
39
+ @sharded_column = column
40
+ end
41
+
42
+ # Accessor for the sharded_columns
43
+ def sharded_column
44
+ @sharded_column
45
+ end
46
+
47
+ # Return a valid Sequel::Dataset that is tied to the shard table and connection for the id and will load values
48
+ # run by the query into the model.
49
+ def shard_for(id)
50
+ result = self.result_for(id)
51
+ ds = result.connection[schema_and_table(result)]
52
+ ds.row_proc = self
53
+ dataset_method_modules.each { |m| ds.instance_eval { extend(m) } }
54
+ ds.model = self
55
+ ds
56
+ end
57
+
58
+ # The result of a lookup for the given id. See Sequel::SchemaSharding::Finder::Result
59
+ def result_for(id)
60
+ Sequel::SchemaSharding::Finder.instance.lookup(self.implicit_table_name, id)
61
+ end
62
+
63
+ # Construct the schema and table for use in a dataset.
64
+ def schema_and_table(result)
65
+ :"#{result.schema}__#{self.implicit_table_name}"
66
+ end
67
+ end
68
+
69
+ # The database connection that has the logical shard.
70
+ def db
71
+ @db ||= finder_result.connection
72
+ end
73
+
74
+ # Wrapper for performing the sharding lookup based on the sharded column.
75
+ def finder_result
76
+ @result ||= self.class.result_for(self.send(self.class.sharded_column))
77
+ end
78
+
79
+ # Dataset instance based on the sharded column.
80
+ def this_server
81
+ @this_server ||= db[self.class.schema_and_table(finder_result)]
82
+ end
83
+
84
+ # Overriden to not use @dataset value from the Sequel::Model. Used internally only.
85
+ def _insert_dataset
86
+ this_server
87
+ end
88
+
89
+ end
90
+ end
91
+ end
@@ -0,0 +1,118 @@
1
+ require 'digest/sha1'
2
+ require 'zlib'
3
+
4
+ module Sequel
5
+ module SchemaSharding
6
+ class Ring
7
+ POINTS_PER_SERVER = 1
8
+
9
+ attr_accessor :shards, :continuum
10
+
11
+ def initialize(shards)
12
+ @shards = shards
13
+ @continuum = nil
14
+ if shards.size > 1
15
+ continuum = []
16
+ shards.each do |shard|
17
+ hash = Digest::SHA1.hexdigest("#{shard}")
18
+ value = Integer("0x#{hash[0..7]}")
19
+ continuum << Entry.new(value, shard)
20
+ end
21
+ @continuum = continuum.sort { |a, b| a.value <=> b.value }
22
+ end
23
+ end
24
+
25
+ def shard_for_id(id)
26
+ if @continuum
27
+ hkey = hash_for(id)
28
+ entryidx = binary_search(@continuum, hkey)
29
+ return @continuum[entryidx].server
30
+ else
31
+ server = @servers.first
32
+ return server if server
33
+ end
34
+
35
+ raise StandardError, "No server available"
36
+ end
37
+
38
+ private
39
+
40
+ def hash_for(key)
41
+ Zlib.crc32(key.to_s)
42
+ end
43
+
44
+ def entry_count
45
+ ((shards.size * POINTS_PER_SERVER)).floor
46
+ end
47
+
48
+ # Native extension to perform the binary search within the continuum
49
+ # space. Fallback to a pure Ruby version if the compilation doesn't work.
50
+ # optional for performance and only necessary if you are using multiple
51
+ # memcached servers.
52
+ begin
53
+ require 'inline'
54
+ inline do |builder|
55
+ builder.c <<-EOM
56
+ int binary_search(VALUE ary, unsigned int r) {
57
+ long upper = RARRAY_LEN(ary) - 1;
58
+ long lower = 0;
59
+ long idx = 0;
60
+ ID value = rb_intern("value");
61
+ VALUE continuumValue;
62
+ unsigned int l;
63
+
64
+ while (lower <= upper) {
65
+ idx = (lower + upper) / 2;
66
+
67
+ continuumValue = rb_funcall(RARRAY_PTR(ary)[idx], value, 0);
68
+ l = NUM2UINT(continuumValue);
69
+ if (l == r) {
70
+ return idx;
71
+ }
72
+ else if (l > r) {
73
+ upper = idx - 1;
74
+ }
75
+ else {
76
+ lower = idx + 1;
77
+ }
78
+ }
79
+ return upper;
80
+ }
81
+ EOM
82
+ end
83
+ rescue LoadError
84
+ # Find the closest index in the Ring with value <= the given value
85
+ def binary_search(ary, value)
86
+ upper = ary.size - 1
87
+ lower = 0
88
+ idx = 0
89
+
90
+ while (lower <= upper) do
91
+ idx = (lower + upper) / 2
92
+ comp = ary[idx].value <=> value
93
+
94
+ if comp == 0
95
+ return idx
96
+ elsif comp > 0
97
+ upper = idx - 1
98
+ else
99
+ lower = idx + 1
100
+ end
101
+ end
102
+ return upper
103
+ end
104
+ end
105
+
106
+ class Entry
107
+ attr_reader :value
108
+ attr_reader :server
109
+
110
+ def initialize(val, srv)
111
+ @value = val
112
+ @server = srv
113
+ end
114
+ end
115
+
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,14 @@
1
+ module Sequel
2
+ module SchemaSharding
3
+ module SequelExt
4
+ module ClassMethods
5
+ def db
6
+ return @db if @db
7
+ end
8
+ end
9
+ end
10
+ end
11
+ end
12
+
13
+ Sequel::Model.plugin Sequel::SchemaSharding::SequelExt
14
+ Sequel::Model.plugin :validation_helpers
@@ -0,0 +1,5 @@
1
+ module Sequel
2
+ module SchemaSharding
3
+ VERSION = "0.0.1"
4
+ end
5
+ end
@@ -0,0 +1,24 @@
1
+ require 'sequel/schema-sharding'
2
+
3
+ namespace :sequel do
4
+ namespace :db do
5
+ desc 'Create databases and shards for tests'
6
+ task :create do
7
+ ENV['RACK_ENV'] ||= 'test'
8
+ Sequel::SchemaSharding.sharding_yml_path = "spec/fixtures/test_db_config.yml"
9
+ Sequel::SchemaSharding.migration_path = "spec/fixtures/db/migrate"
10
+ manager = Sequel::SchemaSharding::DatabaseManager.new
11
+ manager.create_databases
12
+ manager.create_shards
13
+ end
14
+
15
+ desc 'Create databases and shards for tests'
16
+ task :drop do
17
+ ENV['RACK_ENV'] ||= 'test'
18
+ Sequel::SchemaSharding.sharding_yml_path = "spec/fixtures/test_db_config.yml"
19
+ Sequel::SchemaSharding.migration_path = "spec/fixtures/db/migrate"
20
+ manager = Sequel::SchemaSharding::DatabaseManager.new
21
+ manager.drop_databases
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,26 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'sequel/schema-sharding/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "sequel-schema-sharding"
8
+ spec.version = Sequel::SchemaSharding::VERSION
9
+ spec.authors = ["Paul Henry", "James Hart", "Eric Saxby"]
10
+ spec.email = ["dev@wanelo.com"]
11
+ spec.description = %q{}
12
+ spec.summary = %q{Create horizontally sharded Sequel models with Postgres}
13
+ spec.homepage = "https://github.com/wanelo/sequel-schema-sharding"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files`.split($/)
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_dependency "sequel"
22
+ spec.add_dependency "pg"
23
+
24
+ spec.add_development_dependency "bundler", "~> 1.3"
25
+ spec.add_development_dependency "rake"
26
+ end
@@ -0,0 +1,13 @@
1
+ Sequel.migration do
2
+ up do
3
+ create_table(:artists) do
4
+ primary_key :id
5
+ Integer :artist_id
6
+ String :name, :null=>false
7
+ end
8
+ end
9
+
10
+ down do
11
+ drop_table(:artists)
12
+ end
13
+ end
@@ -0,0 +1,12 @@
1
+ Sequel.migration do
2
+ up do
3
+ create_table(:boofs) do
4
+ primary_key :id
5
+ String :name, :null=>false
6
+ end
7
+ end
8
+
9
+ down do
10
+ drop_table(:boofs)
11
+ end
12
+ end
@@ -0,0 +1,67 @@
1
+ test:
2
+ tables:
3
+ artists:
4
+ schema_name: sequel_logical_artists_%e_%s
5
+ logical_shards:
6
+ 1..10: shard1
7
+ 11..20: shard2
8
+ boof:
9
+ schema_name: sequel_logical_boof_%e_%s
10
+ logical_shards:
11
+ 1..10: shard1
12
+ 11..20: shard2
13
+ physical_shards:
14
+ shard1:
15
+ host: 127.0.0.1
16
+ database: sequel_test_shard1
17
+ shard2:
18
+ host: 127.0.0.1
19
+ database: sequel_test_shard2
20
+ common:
21
+ username: postgres
22
+ password: boomboomkaboom
23
+ boom:
24
+ tables:
25
+ artists:
26
+ schema_name: sequel_explosions_artists_%e_%s
27
+ logical_shards:
28
+ 1..10: shard1
29
+ 11..20: shard2
30
+ boof:
31
+ schema_name: sequel_explosions_boof_%e_%s
32
+ logical_shards:
33
+ 1..10: shard1
34
+ 11..20: shard2
35
+ physical_shards:
36
+ shard1:
37
+ host: 127.0.0.1
38
+ database: sequel_boom_shard1
39
+ shard2:
40
+ host: 127.0.0.1
41
+ database: sequel_boom_shard2
42
+ common:
43
+ username: postgres
44
+ password: boomboomkaboom
45
+ port: 5432
46
+ #boom:
47
+ # logical_shards:
48
+ # user_saves:
49
+ # 1..10: shard1
50
+ # 11..20: shard2
51
+ # product_saves:
52
+ # 1..10: shard2
53
+ # 11..20: shard3
54
+ # physical_shards:
55
+ # shard1:
56
+ # host: 127.0.0.1
57
+ # database: wanelo_saves_boom_shard1
58
+ # shard2:
59
+ # host: 127.0.0.1
60
+ # database: wanelo_saves_boom_shard2
61
+ # shard3:
62
+ # host: 127.0.0.1
63
+ # database: wanelo_saves_boom_shard3
64
+ # common:
65
+ # username: postgres
66
+ # password: boomboomkaboom
67
+ # port: 5432
@@ -0,0 +1,42 @@
1
+ require 'spec_helper'
2
+ require 'sequel/schema-sharding/configuration'
3
+
4
+ describe Sequel::SchemaSharding::Configuration do
5
+
6
+ let(:config) { Sequel::SchemaSharding::Configuration.new(:boom, 'spec/fixtures/test_db_config.yml') }
7
+
8
+ describe '#logical_shard_configs' do
9
+ it 'returns a hash representing the mapping between logical and physical shards' do
10
+ shards = config.logical_shard_configs('boof')
11
+
12
+ expect(shards.length).to eq(20)
13
+
14
+ shards.each_pair do |key, value|
15
+ if key <= 10
16
+ expect(value).to eq('shard1')
17
+ elsif key > 10
18
+ expect(value).to eq('shard2')
19
+ end
20
+ end
21
+ end
22
+ end
23
+
24
+ describe '#physical_shard_configs' do
25
+ it 'returns a hash representing the configuration for all physical shards' do
26
+ shards = config.physical_shard_configs
27
+
28
+ expect(shards.length).to eq(2)
29
+
30
+ i = 1
31
+ shards.each_pair do |key, value|
32
+ expect(value['host']).to eq('127.0.0.1')
33
+ expect(value['database']).to eq("sequel_boom_shard#{i}")
34
+ expect(value['username']).to eq('postgres')
35
+ expect(value['password']).to eq('boomboomkaboom')
36
+ expect(value['port']).to eq(5432)
37
+
38
+ i += 1
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,33 @@
1
+ require 'spec_helper'
2
+ require 'sequel/schema-sharding/connection_manager'
3
+
4
+ describe Sequel::SchemaSharding::ConnectionManager do
5
+ let(:config) { Sequel::SchemaSharding::Configuration.new('boom', 'spec/fixtures/test_db_config.yml') }
6
+
7
+ subject { Sequel::SchemaSharding::ConnectionManager.new }
8
+
9
+ before { subject.stubs(:config).returns(config) }
10
+
11
+ describe '#[]' do
12
+ it 'returns a valid connection instance for the specified physical shard' do
13
+ expect(subject['shard1']).to be_a(Sequel::Postgres::Database)
14
+ expect(subject['shard2']).to be_a(Sequel::Postgres::Database)
15
+ end
16
+ end
17
+
18
+ describe "#schema_for" do
19
+ it "returns the schema name based on env and shard number" do
20
+ subject.schema_for('boof', 'pickles', 3).should eq 'sequel_explosions_boof_pickles_3'
21
+ end
22
+ end
23
+
24
+ describe "#default_dataset_for" do
25
+ it "returns a dataset scoped to a configured schema" do
26
+ # TODO ConnectionManager is dependent on global state from Sequel::SchemaSharding.config.
27
+ # This should be deconstructed to allow for injection of a mock config for testing.
28
+ dataset = subject.default_dataset_for("artists")
29
+ expect(dataset).to be_a(Sequel::Dataset)
30
+ expect(dataset.first_source_table).to eql(:'sequel_explosions_artists_test_1__artists')
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,124 @@
1
+ require 'spec_helper'
2
+ require 'sequel/schema-sharding'
3
+
4
+ describe Sequel::SchemaSharding::DatabaseManager, type: :manager, sharded: true do
5
+
6
+ let(:config) { Sequel::SchemaSharding::Configuration.new('boom', 'spec/fixtures/test_db_config.yml') }
7
+
8
+ after do
9
+ Sequel::SchemaSharding.connection_manager.disconnect
10
+ end
11
+
12
+ around do |ex|
13
+ Sequel::SchemaSharding.stubs(:config).returns(config)
14
+
15
+ @manager = Sequel::SchemaSharding::DatabaseManager.new
16
+ @manager.send(:connection_manager).disconnect
17
+ DatabaseHelper.disconnect
18
+
19
+ DatabaseHelper.drop_db('sequel_boom_shard1')
20
+ DatabaseHelper.drop_db('sequel_boom_shard2')
21
+
22
+ ex.call
23
+ end
24
+
25
+ describe '#create_database' do
26
+ context 'database does not exist' do
27
+ it 'creates the database for the current environment' do
28
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
29
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
30
+ @manager.create_databases
31
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
32
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
33
+ end
34
+ end
35
+
36
+ context 'database exists' do
37
+
38
+ before(:each) do
39
+ @manager.create_databases
40
+ end
41
+
42
+ it 'outputs message to stderr' do
43
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
44
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
45
+ $stderr.expects(:puts).with(regexp_matches(/already exists/)).twice
46
+ @manager.create_databases
47
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
48
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
49
+ end
50
+ end
51
+ end
52
+
53
+ describe '#drop_databases' do
54
+ context 'databases exist' do
55
+ it 'drops the database for the current environment' do
56
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
57
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
58
+
59
+ @manager.create_databases
60
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
61
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
62
+
63
+ @manager.drop_databases
64
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
65
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
66
+ end
67
+ end
68
+
69
+ context 'databases dont exist' do
70
+ it 'raises an error' do
71
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
72
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
73
+
74
+ $stderr.expects(:puts).with(regexp_matches(/database doesnt exist/)).times(2)
75
+
76
+ @manager.drop_databases
77
+ end
78
+ end
79
+ end
80
+
81
+ describe 'logical shards' do
82
+ before(:each) do
83
+ @manager.create_databases
84
+ end
85
+
86
+ describe '#create_shards' do
87
+ it 'creates the database structure' do
88
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
89
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
90
+ @manager.create_shards
91
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
92
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
93
+ end
94
+
95
+ context 'shards already exist' do
96
+ it 'prints that shards already exist' do
97
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
98
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
99
+ @manager.create_shards
100
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
101
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
102
+
103
+ $stderr.expects(:puts).with(regexp_matches(/already exists/)).at_least_once
104
+ @manager.create_shards
105
+ end
106
+ end
107
+ end
108
+
109
+ describe '#drop_schemas' do
110
+ context 'schemas exist' do
111
+ it 'drops the schemas' do
112
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
113
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
114
+ @manager.create_shards
115
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
116
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
117
+ @manager.drop_shards
118
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
119
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
120
+ end
121
+ end
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,13 @@
1
+ require 'spec_helper'
2
+ require 'sequel/schema-sharding'
3
+
4
+ describe Sequel::SchemaSharding::Finder do
5
+
6
+ describe '.lookup' do
7
+ it 'returns an object with a valid connection and schema' do
8
+ result = Sequel::SchemaSharding::Finder.instance.lookup('boof', 60)
9
+ expect(result.connection).to be_a(Sequel::Postgres::Database)
10
+ expect(result.schema).to eq('sequel_logical_boof_test_2')
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,40 @@
1
+ require 'spec_helper'
2
+
3
+ describe Sequel::SchemaSharding, 'Model' do
4
+
5
+ let(:model) do
6
+ klass = Sequel::SchemaSharding::Model('artists')
7
+ klass.instance_eval do
8
+ set_sharded_column :artist_id
9
+ set_columns [:artist_id, :name]
10
+
11
+ def by_id(id)
12
+ shard_for(id).where(artist_id: id)
13
+ end
14
+
15
+ def self.implicit_table_name
16
+ 'artists'
17
+ end
18
+ end
19
+ klass
20
+ end
21
+
22
+ describe '#by_id' do
23
+ it 'returns a valid artist by id' do
24
+ artist = model.create(artist_id: 14, name: 'Paul')
25
+ expect(artist.id).to_not be_nil
26
+ read_back_artist = model.by_id(14).first
27
+ expect(read_back_artist).to be_a(model)
28
+ expect(read_back_artist.name).to eql('Paul')
29
+ end
30
+ end
31
+
32
+ describe '#create' do
33
+ it 'creates a valid artist' do
34
+ artist = model.create(artist_id: 234, name: 'Paul')
35
+ expect(artist).to be_a(model)
36
+ expect(artist.name).to eql('Paul')
37
+ end
38
+ end
39
+
40
+ end
@@ -0,0 +1,25 @@
1
+ require 'spec_helper'
2
+ require 'sequel/schema-sharding'
3
+
4
+ describe Sequel::SchemaSharding::Ring do
5
+
6
+ describe 'a ring of servers' do
7
+ it 'have the continuum sorted by value' do
8
+ shards = [1, 2, 3, 4, 5, 6, 7, 8]
9
+ ring = Sequel::SchemaSharding::Ring.new(shards)
10
+ previous_value = 0
11
+ ring.continuum.each do |entry|
12
+ expect(entry.value).to be > previous_value
13
+ previous_value = entry.value
14
+ end
15
+ end
16
+ end
17
+
18
+ describe '#shard_for_id' do
19
+ it 'returns a server for a given id' do
20
+ shards = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
21
+ ring = Sequel::SchemaSharding::Ring.new(shards)
22
+ expect(ring.shard_for_id(3489409)).to eq(4)
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,53 @@
1
+ ENV['RACK_ENV'] ||= 'test'
2
+
3
+ require 'bundler/setup'
4
+ Bundler.require 'test'
5
+
6
+ require 'sequel-schema-sharding'
7
+ require 'support/database_helper'
8
+ require 'mocha/api'
9
+
10
+ RSpec.configure do |config|
11
+ config.treat_symbols_as_metadata_keys_with_true_values = true
12
+ config.run_all_when_everything_filtered = true
13
+ config.filter_run :focus
14
+ config.alias_example_to :fit, focus: true
15
+ config.mock_framework = :mocha
16
+
17
+
18
+ # Run specs in random order to surface order dependencies. If you find an
19
+ # order dependency and want to debug it, you can fix the order by providing
20
+ # the seed, which is printed after each run.
21
+ # --seed 1234
22
+ config.order = 'random'
23
+
24
+ config.before :all do |ex|
25
+ Sequel::SchemaSharding.logger = Logger.new(StringIO.new)
26
+ Sequel::SchemaSharding.sharding_yml_path = "spec/fixtures/test_db_config.yml"
27
+ Sequel::SchemaSharding.migration_path = "spec/fixtures/db/migrate"
28
+ end
29
+
30
+ config.around :each do |ex|
31
+ #Sequel::SchemaSharding.config = Sequel::SchemaSharding::Configuration.new('boom', 'spec/fixtures/test_db_config.yml')
32
+
33
+ # Start transactions in each connection to the physical shards
34
+ connections = Sequel::SchemaSharding.config.physical_shard_configs.map do |shard_config|
35
+ Sequel::SchemaSharding.connection_manager[shard_config[0]]
36
+ end
37
+
38
+ start_transaction_proc = Proc.new do |connections|
39
+ if connections.length == 0
40
+ ex.run
41
+ else
42
+ connections[0].transaction do
43
+ connections.shift
44
+ start_transaction_proc.call(connections)
45
+ raise Sequel::Rollback
46
+ end
47
+ end
48
+ end
49
+
50
+ start_transaction_proc.call(connections)
51
+ end
52
+
53
+ end
@@ -0,0 +1,35 @@
1
+ require 'sequel'
2
+
3
+ class DatabaseHelper
4
+ def self.db
5
+ @db = Sequel.postgres(user: 'postgres', password: 'alkfjasdlfj', host: 'localhost')
6
+ end
7
+
8
+ def self.disconnect
9
+ db.disconnect
10
+ end
11
+
12
+ def self.drop_db(database)
13
+ if db_exists?("#{database}")
14
+ self.disconnect
15
+
16
+ db.run("DROP DATABASE #{database}")
17
+ end
18
+ end
19
+
20
+ def self.db_exists?(database)
21
+ dbs.include?(database)
22
+ end
23
+
24
+ def self.dbs
25
+ db.fetch('SELECT datname FROM pg_database WHERE datistemplate = false;').all.map { |d| d[:datname] }
26
+ end
27
+
28
+ def self.schema_exists?(database, schema)
29
+ schemas(database).include?(schema)
30
+ end
31
+
32
+ def self.schemas(database)
33
+ Sequel::SchemaSharding.connection_manager[database].fetch('select nspname from pg_catalog.pg_namespace;').all.map { |d| d[:nspname] }
34
+ end
35
+ end
metadata ADDED
@@ -0,0 +1,143 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sequel-schema-sharding
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Paul Henry
8
+ - James Hart
9
+ - Eric Saxby
10
+ autorequire:
11
+ bindir: bin
12
+ cert_chain: []
13
+ date: 2013-09-06 00:00:00.000000000 Z
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: sequel
17
+ requirement: !ruby/object:Gem::Requirement
18
+ requirements:
19
+ - - '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ requirements:
26
+ - - '>='
27
+ - !ruby/object:Gem::Version
28
+ version: '0'
29
+ - !ruby/object:Gem::Dependency
30
+ name: pg
31
+ requirement: !ruby/object:Gem::Requirement
32
+ requirements:
33
+ - - '>='
34
+ - !ruby/object:Gem::Version
35
+ version: '0'
36
+ type: :runtime
37
+ prerelease: false
38
+ version_requirements: !ruby/object:Gem::Requirement
39
+ requirements:
40
+ - - '>='
41
+ - !ruby/object:Gem::Version
42
+ version: '0'
43
+ - !ruby/object:Gem::Dependency
44
+ name: bundler
45
+ requirement: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - ~>
48
+ - !ruby/object:Gem::Version
49
+ version: '1.3'
50
+ type: :development
51
+ prerelease: false
52
+ version_requirements: !ruby/object:Gem::Requirement
53
+ requirements:
54
+ - - ~>
55
+ - !ruby/object:Gem::Version
56
+ version: '1.3'
57
+ - !ruby/object:Gem::Dependency
58
+ name: rake
59
+ requirement: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - '>='
62
+ - !ruby/object:Gem::Version
63
+ version: '0'
64
+ type: :development
65
+ prerelease: false
66
+ version_requirements: !ruby/object:Gem::Requirement
67
+ requirements:
68
+ - - '>='
69
+ - !ruby/object:Gem::Version
70
+ version: '0'
71
+ description: ''
72
+ email:
73
+ - dev@wanelo.com
74
+ executables: []
75
+ extensions: []
76
+ extra_rdoc_files: []
77
+ files:
78
+ - .gitignore
79
+ - .rspec
80
+ - .travis.yml
81
+ - Gemfile
82
+ - LICENSE.txt
83
+ - README.md
84
+ - Rakefile
85
+ - lib/sequel-schema-sharding.rb
86
+ - lib/sequel/schema-sharding.rb
87
+ - lib/sequel/schema-sharding/configuration.rb
88
+ - lib/sequel/schema-sharding/connection_manager.rb
89
+ - lib/sequel/schema-sharding/database_manager.rb
90
+ - lib/sequel/schema-sharding/finder.rb
91
+ - lib/sequel/schema-sharding/model.rb
92
+ - lib/sequel/schema-sharding/ring.rb
93
+ - lib/sequel/schema-sharding/sequel_ext.rb
94
+ - lib/sequel/schema-sharding/version.rb
95
+ - lib/sequel/tasks/test.rake
96
+ - sequel-schema-sharding.gemspec
97
+ - spec/fixtures/db/migrate/artists/001_create_test_table.rb
98
+ - spec/fixtures/db/migrate/boof/001_create_boof_table.rb
99
+ - spec/fixtures/test_db_config.yml
100
+ - spec/schema-sharding/configuration_spec.rb
101
+ - spec/schema-sharding/connection_manager_spec.rb
102
+ - spec/schema-sharding/database_manager_spec.rb
103
+ - spec/schema-sharding/finder_spec.rb
104
+ - spec/schema-sharding/model_spec.rb
105
+ - spec/schema-sharding/ring_spec.rb
106
+ - spec/spec_helper.rb
107
+ - spec/support/database_helper.rb
108
+ homepage: https://github.com/wanelo/sequel-schema-sharding
109
+ licenses:
110
+ - MIT
111
+ metadata: {}
112
+ post_install_message:
113
+ rdoc_options: []
114
+ require_paths:
115
+ - lib
116
+ required_ruby_version: !ruby/object:Gem::Requirement
117
+ requirements:
118
+ - - '>='
119
+ - !ruby/object:Gem::Version
120
+ version: '0'
121
+ required_rubygems_version: !ruby/object:Gem::Requirement
122
+ requirements:
123
+ - - '>='
124
+ - !ruby/object:Gem::Version
125
+ version: '0'
126
+ requirements: []
127
+ rubyforge_project:
128
+ rubygems_version: 2.0.7
129
+ signing_key:
130
+ specification_version: 4
131
+ summary: Create horizontally sharded Sequel models with Postgres
132
+ test_files:
133
+ - spec/fixtures/db/migrate/artists/001_create_test_table.rb
134
+ - spec/fixtures/db/migrate/boof/001_create_boof_table.rb
135
+ - spec/fixtures/test_db_config.yml
136
+ - spec/schema-sharding/configuration_spec.rb
137
+ - spec/schema-sharding/connection_manager_spec.rb
138
+ - spec/schema-sharding/database_manager_spec.rb
139
+ - spec/schema-sharding/finder_spec.rb
140
+ - spec/schema-sharding/model_spec.rb
141
+ - spec/schema-sharding/ring_spec.rb
142
+ - spec/spec_helper.rb
143
+ - spec/support/database_helper.rb