sequel-schema-sharding 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 9a83484015fdcd8dc4799e752d6d82ee94498738
4
+ data.tar.gz: 2e44e0d65a6c598371035daddc1724022b3c893b
5
+ SHA512:
6
+ metadata.gz: bac9d7a0ed39dbe558e2f0bd6422bda34f68676662ba5283722340c1ffbf784f6e8c433ca6b118efe41c8a0e66ed123c8bfd2f482597ace1ab750a2b7d2fcbd1
7
+ data.tar.gz: 724cae47d487ee6e63a8e969ae8ed9f1d380ffaf61b3877074578a9704565fd0290ffd27550904afc9ad3949aba536b4fa69d5209670a39b13f21ae2e0fd874c
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --color
2
+ --format progress
data/.travis.yml ADDED
@@ -0,0 +1,8 @@
1
+ language: ruby
2
+ rvm:
3
+ - 1.9.3
4
+ - 2.0.0-p247
5
+ before_script: "bundle exec rake sequel:db:create"
6
+ script: "bundle exec rspec"
7
+ notifications:
8
+ email: false
data/Gemfile ADDED
@@ -0,0 +1,10 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in sequel-sharding.gemspec
4
+ gemspec
5
+
6
+ group :test do
7
+ gem 'rspec'
8
+ gem 'mocha', require: false
9
+ gem 'pry-nav'
10
+ end
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 TODO: Write your name
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,33 @@
1
+ Sequel::SchemaSharding
2
+ ================
3
+
4
+ [![Build Status](https://travis-ci.org/wanelo/sequel-sharding.png?branch=master)](https://travis-ci.org/wanelo/sequel-sharding)
5
+
6
+ Horizontally shard postgres with the Sequel gem. This gem allows you to configure mappings between logical and
7
+ physical shards, and pool connections between logical shards on the same physical server.
8
+
9
+ ## Installation
10
+
11
+ Add this line to your application's Gemfile:
12
+
13
+ gem 'sequel-sharding'
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install sequel-sharding
22
+
23
+ ## Usage
24
+
25
+ TODO :)
26
+
27
+ ## Contributing
28
+
29
+ 1. Fork it
30
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
31
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
32
+ 4. Push to the branch (`git push origin my-new-feature`)
33
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require "bundler/gem_tasks"
2
+ import 'lib/sequel/tasks/test.rake'
@@ -0,0 +1 @@
1
+ require 'sequel/schema-sharding'
@@ -0,0 +1,54 @@
1
+ require 'sequel/schema-sharding/version'
2
+ require 'sequel/schema-sharding/configuration'
3
+ require 'sequel/schema-sharding/connection_manager'
4
+ require 'sequel/schema-sharding/database_manager'
5
+ require 'sequel/schema-sharding/ring'
6
+ require 'sequel/schema-sharding/finder'
7
+ require 'sequel/schema-sharding/sequel_ext'
8
+ require 'sequel/schema-sharding/model'
9
+ require 'logger'
10
+
11
+ module Sequel
12
+ module SchemaSharding
13
+ def self.config
14
+ @config ||= Sequel::SchemaSharding::Configuration.new(ENV['RACK_ENV'], sharding_yml_path)
15
+ end
16
+
17
+ def self.config=(config)
18
+ @config = config
19
+ end
20
+
21
+ def self.logger
22
+ @logger ||= Logger.new($stdout)
23
+ end
24
+
25
+ def self.logger=(logger)
26
+ @logger = logger
27
+ end
28
+
29
+ def self.connection_manager
30
+ @connection_manager ||= ConnectionManager.new
31
+ end
32
+
33
+ def self.connection_manager=(connection_manager)
34
+ @connection_manager = connection_manager
35
+ end
36
+
37
+ def self.sharding_yml_path
38
+ @sharding_yml_path ||= File.expand_path('../../../config/sharding.yml', __FILE__)
39
+ end
40
+
41
+ def self.sharding_yml_path=(path)
42
+ @sharding_yml_path = path
43
+ end
44
+
45
+ def self.migration_path
46
+ @migration_path || raise('You must set the migration path.')
47
+ end
48
+
49
+ def self.migration_path=(path)
50
+ @migration_path = path
51
+ end
52
+
53
+ end
54
+ end
@@ -0,0 +1,54 @@
1
+ require 'yaml'
2
+
3
+ module Sequel
4
+ module SchemaSharding
5
+ class Configuration
6
+ attr_reader :env, :yaml_path
7
+
8
+ def initialize(env, yaml_path)
9
+ @env = env
10
+ @yaml_path = yaml_path
11
+ end
12
+
13
+ def physical_shard_configs
14
+ @physical_shard_configs ||= config['physical_shards'].inject({}) do |hash, value|
15
+ hash[value[0]] = config['common'].merge(value[1])
16
+ hash
17
+ end
18
+ end
19
+
20
+ def logical_shard_configs(table_name)
21
+ table_name = table_name.to_s
22
+ @logical_shard_table_configs ||= {}
23
+ @logical_shard_table_configs[table_name] ||= begin
24
+ table_configs = config['tables'][table_name]
25
+ raise "Unknown table #{table_name} in configuration" if table_configs.nil?
26
+ table_configs['logical_shards'].inject({}) do |hash, value|
27
+ eval(value[0]).each do |i|
28
+ hash[i] = value[1]
29
+ end
30
+ hash
31
+ end
32
+ end
33
+ end
34
+
35
+ def table_names
36
+ config['tables'].keys
37
+ end
38
+
39
+ def schema_name(table_name)
40
+ config['tables'][table_name.to_s]['schema_name']
41
+ end
42
+
43
+ private
44
+
45
+ def config
46
+ yaml[env.to_s]
47
+ end
48
+
49
+ def yaml
50
+ @raw_yaml ||= YAML.load_file(yaml_path)
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,48 @@
1
+ require 'singleton'
2
+
3
+ module Sequel
4
+ module SchemaSharding
5
+ class ConnectionManager
6
+ attr_reader :connections
7
+
8
+ def initialize
9
+ @connections = {}
10
+ end
11
+
12
+ def [](name)
13
+ config = db_config_for(name)
14
+ @connections[name.to_s] ||= Sequel.postgres(:user => config['username'],
15
+ :password => config['password'],
16
+ :host => config['host'],
17
+ :database => config['database'])
18
+ end
19
+
20
+ def disconnect
21
+ @connections.each_value do |conn|
22
+ conn.disconnect
23
+ end
24
+ @connections = {}
25
+ end
26
+
27
+ def schema_for(table_name, environment, shard_number)
28
+ config.schema_name(table_name).gsub('%e', environment).gsub('%s', shard_number.to_s)
29
+ end
30
+
31
+ def default_dataset_for(table_name)
32
+ shard_number = config.logical_shard_configs(table_name).keys.first
33
+ shard_name = config.logical_shard_configs(table_name)[shard_number]
34
+ self[shard_name][:"#{schema_for(table_name, ENV['RACK_ENV'], shard_number)}__#{table_name}"]
35
+ end
36
+
37
+ private
38
+
39
+ def db_config_for(name)
40
+ config.physical_shard_configs[name]
41
+ end
42
+
43
+ def config
44
+ Sequel::SchemaSharding.config
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,105 @@
1
+ require 'sequel'
2
+ require 'sequel/schema-sharding/connection_manager'
3
+
4
+ Sequel.extension :migration
5
+
6
+ module Sequel
7
+ module SchemaSharding
8
+ class DatabaseManager
9
+ def create_databases
10
+ config.physical_shard_configs.each_pair do |name, config|
11
+ begin
12
+ # Need to create connection manually with specifying a database in order to create the database
13
+ connection = Sequel.postgres(:user => config['username'],
14
+ :password => config['password'],
15
+ :host => config['host'])
16
+
17
+ Sequel::SchemaSharding.logger.info "Creating #{config['database']}.."
18
+
19
+ connection.run("CREATE DATABASE #{config['database']}")
20
+ rescue Sequel::DatabaseError => e
21
+ if e.message.include?('already exists')
22
+ $stderr.puts "#{config['database']} database already exists"
23
+ else
24
+ raise e
25
+ end
26
+ ensure
27
+ connection.disconnect
28
+ end
29
+ end
30
+ end
31
+
32
+ def drop_databases
33
+ connection_manager.disconnect
34
+ config.physical_shard_configs.each_pair do |name, config|
35
+ # Need to create connection manually with specifying a database in order to create the database
36
+ begin
37
+ connection = Sequel.postgres(:user => config['username'],
38
+ :password => config['password'],
39
+ :host => config['host'])
40
+
41
+ Sequel::SchemaSharding.logger.info "Dropping #{config['database']}.."
42
+ connection.run("DROP DATABASE #{config['database']}")
43
+ rescue Sequel::DatabaseError => e
44
+ if e.message.include?('does not exist')
45
+ $stderr.puts "#{config['database']} database doesnt exist"
46
+ else
47
+ raise e
48
+ end
49
+ ensure
50
+ connection.disconnect
51
+ end
52
+ end
53
+ end
54
+
55
+ def create_shards
56
+ config.table_names.each do |table_name|
57
+ config.logical_shard_configs(table_name).each_pair do |shard_number, physical_shard|
58
+ schema_name = connection_manager.schema_for(table_name, env, shard_number)
59
+ Sequel::SchemaSharding.logger.info "Creating schema #{schema_name} on #{physical_shard}.."
60
+ connection = connection_manager[physical_shard]
61
+
62
+ begin
63
+ connection.run("CREATE SCHEMA #{schema_name}")
64
+ rescue Sequel::DatabaseError => e
65
+ if e.message.include?('already exists')
66
+ $stderr.puts "#{schema_name} schema already exists"
67
+ else
68
+ raise e
69
+ end
70
+ end
71
+
72
+ connection.run("SET search_path TO #{schema_name}")
73
+
74
+ Sequel::Migrator.run(connection, Sequel::SchemaSharding.migration_path + "/#{table_name}", :use_transactions => true)
75
+ end
76
+ end
77
+ end
78
+
79
+ def drop_shards
80
+ config.table_names.each do |table_name|
81
+ config.logical_shard_configs(table_name).each_pair do |shard_number, physical_shard|
82
+ schema_name = connection_manager.schema_for(table_name, env, shard_number)
83
+ Sequel::SchemaSharding.logger.info "Dropping schema #{schema_name} on #{physical_shard}.."
84
+ connection = connection_manager[physical_shard]
85
+ connection.run("DROP SCHEMA #{schema_name} CASCADE")
86
+ end
87
+ end
88
+ end
89
+
90
+ private
91
+
92
+ def env
93
+ config.env
94
+ end
95
+
96
+ def config
97
+ Sequel::SchemaSharding.config
98
+ end
99
+
100
+ def connection_manager
101
+ Sequel::SchemaSharding.connection_manager
102
+ end
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,43 @@
1
+ require 'singleton'
2
+
3
+ module Sequel
4
+ module SchemaSharding
5
+ class Finder
6
+ class Result
7
+ attr_reader :connection, :schema
8
+
9
+ def initialize(connection, schema)
10
+ @connection = connection
11
+ @schema = schema
12
+ end
13
+ end
14
+
15
+ include ::Singleton
16
+
17
+ def lookup(table_name, id)
18
+ shard_number = shard_for_id(table_name, id)
19
+ physical_shard = config.logical_shard_configs(table_name)[shard_number]
20
+
21
+ conn = Sequel::SchemaSharding.connection_manager[physical_shard]
22
+ schema = Sequel::SchemaSharding.connection_manager.schema_for(table_name, config.env, shard_number)
23
+
24
+ Result.new(conn, schema)
25
+ end
26
+
27
+ private
28
+
29
+ def shard_for_id(table_name, id)
30
+ ring(table_name).shard_for_id(id)
31
+ end
32
+
33
+ def ring(table_name)
34
+ @rings ||= {}
35
+ @rings[table_name] ||= Sequel::SchemaSharding::Ring.new(config.logical_shard_configs(table_name).keys)
36
+ end
37
+
38
+ def config
39
+ Sequel::SchemaSharding.config
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,91 @@
1
+ require 'sequel'
2
+
3
+ module Sequel
4
+ module SchemaSharding
5
+ # Extensions to the Sequel model to allow logical/physical shards. Actual table models should
6
+ # inherit this class like so:
7
+ #
8
+ # class Cat < Sequel::SchemaSharding::Model
9
+ # set_columns [:cat_id, :fur, :tongue, :whiskers] # Columns in the database need to be predefined.
10
+ # set_sharded_column :cat_id # Define the shard column
11
+ #
12
+ # def self.by_cat_id(id)
13
+ # # You should always call shard_for in finders to select the correct connection.
14
+ # shard_for(id).where(cat_id: id)
15
+ # end
16
+ # end
17
+
18
+ def self.Model(source)
19
+ klass = Sequel::Model(Sequel::SchemaSharding.connection_manager.default_dataset_for(source))
20
+
21
+ klass.include(SchemaSharding::ShardedModel)
22
+
23
+ klass
24
+ end
25
+
26
+ module ShardedModel
27
+
28
+ def self.included(base)
29
+ base.extend(ClassMethods)
30
+ end
31
+
32
+ module ClassMethods
33
+
34
+ #protected
35
+
36
+ # Set the column on which the current model is sharded. This is used when saving, inserting and finding
37
+ # to decide which connection to use.
38
+ def set_sharded_column(column)
39
+ @sharded_column = column
40
+ end
41
+
42
+ # Accessor for the sharded_columns
43
+ def sharded_column
44
+ @sharded_column
45
+ end
46
+
47
+ # Return a valid Sequel::Dataset that is tied to the shard table and connection for the id and will load values
48
+ # run by the query into the model.
49
+ def shard_for(id)
50
+ result = self.result_for(id)
51
+ ds = result.connection[schema_and_table(result)]
52
+ ds.row_proc = self
53
+ dataset_method_modules.each { |m| ds.instance_eval { extend(m) } }
54
+ ds.model = self
55
+ ds
56
+ end
57
+
58
+ # The result of a lookup for the given id. See Sequel::SchemaSharding::Finder::Result
59
+ def result_for(id)
60
+ Sequel::SchemaSharding::Finder.instance.lookup(self.implicit_table_name, id)
61
+ end
62
+
63
+ # Construct the schema and table for use in a dataset.
64
+ def schema_and_table(result)
65
+ :"#{result.schema}__#{self.implicit_table_name}"
66
+ end
67
+ end
68
+
69
+ # The database connection that has the logical shard.
70
+ def db
71
+ @db ||= finder_result.connection
72
+ end
73
+
74
+ # Wrapper for performing the sharding lookup based on the sharded column.
75
+ def finder_result
76
+ @result ||= self.class.result_for(self.send(self.class.sharded_column))
77
+ end
78
+
79
+ # Dataset instance based on the sharded column.
80
+ def this_server
81
+ @this_server ||= db[self.class.schema_and_table(finder_result)]
82
+ end
83
+
84
+ # Overriden to not use @dataset value from the Sequel::Model. Used internally only.
85
+ def _insert_dataset
86
+ this_server
87
+ end
88
+
89
+ end
90
+ end
91
+ end
@@ -0,0 +1,118 @@
1
+ require 'digest/sha1'
2
+ require 'zlib'
3
+
4
+ module Sequel
5
+ module SchemaSharding
6
+ class Ring
7
+ POINTS_PER_SERVER = 1
8
+
9
+ attr_accessor :shards, :continuum
10
+
11
+ def initialize(shards)
12
+ @shards = shards
13
+ @continuum = nil
14
+ if shards.size > 1
15
+ continuum = []
16
+ shards.each do |shard|
17
+ hash = Digest::SHA1.hexdigest("#{shard}")
18
+ value = Integer("0x#{hash[0..7]}")
19
+ continuum << Entry.new(value, shard)
20
+ end
21
+ @continuum = continuum.sort { |a, b| a.value <=> b.value }
22
+ end
23
+ end
24
+
25
+ def shard_for_id(id)
26
+ if @continuum
27
+ hkey = hash_for(id)
28
+ entryidx = binary_search(@continuum, hkey)
29
+ return @continuum[entryidx].server
30
+ else
31
+ server = @servers.first
32
+ return server if server
33
+ end
34
+
35
+ raise StandardError, "No server available"
36
+ end
37
+
38
+ private
39
+
40
+ def hash_for(key)
41
+ Zlib.crc32(key.to_s)
42
+ end
43
+
44
+ def entry_count
45
+ ((shards.size * POINTS_PER_SERVER)).floor
46
+ end
47
+
48
+ # Native extension to perform the binary search within the continuum
49
+ # space. Fallback to a pure Ruby version if the compilation doesn't work.
50
+ # optional for performance and only necessary if you are using multiple
51
+ # memcached servers.
52
+ begin
53
+ require 'inline'
54
+ inline do |builder|
55
+ builder.c <<-EOM
56
+ int binary_search(VALUE ary, unsigned int r) {
57
+ long upper = RARRAY_LEN(ary) - 1;
58
+ long lower = 0;
59
+ long idx = 0;
60
+ ID value = rb_intern("value");
61
+ VALUE continuumValue;
62
+ unsigned int l;
63
+
64
+ while (lower <= upper) {
65
+ idx = (lower + upper) / 2;
66
+
67
+ continuumValue = rb_funcall(RARRAY_PTR(ary)[idx], value, 0);
68
+ l = NUM2UINT(continuumValue);
69
+ if (l == r) {
70
+ return idx;
71
+ }
72
+ else if (l > r) {
73
+ upper = idx - 1;
74
+ }
75
+ else {
76
+ lower = idx + 1;
77
+ }
78
+ }
79
+ return upper;
80
+ }
81
+ EOM
82
+ end
83
+ rescue LoadError
84
+ # Find the closest index in the Ring with value <= the given value
85
+ def binary_search(ary, value)
86
+ upper = ary.size - 1
87
+ lower = 0
88
+ idx = 0
89
+
90
+ while (lower <= upper) do
91
+ idx = (lower + upper) / 2
92
+ comp = ary[idx].value <=> value
93
+
94
+ if comp == 0
95
+ return idx
96
+ elsif comp > 0
97
+ upper = idx - 1
98
+ else
99
+ lower = idx + 1
100
+ end
101
+ end
102
+ return upper
103
+ end
104
+ end
105
+
106
+ class Entry
107
+ attr_reader :value
108
+ attr_reader :server
109
+
110
+ def initialize(val, srv)
111
+ @value = val
112
+ @server = srv
113
+ end
114
+ end
115
+
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,14 @@
1
+ module Sequel
2
+ module SchemaSharding
3
+ module SequelExt
4
+ module ClassMethods
5
+ def db
6
+ return @db if @db
7
+ end
8
+ end
9
+ end
10
+ end
11
+ end
12
+
13
+ Sequel::Model.plugin Sequel::SchemaSharding::SequelExt
14
+ Sequel::Model.plugin :validation_helpers
@@ -0,0 +1,5 @@
1
+ module Sequel
2
+ module SchemaSharding
3
+ VERSION = "0.0.1"
4
+ end
5
+ end
@@ -0,0 +1,24 @@
1
+ require 'sequel/schema-sharding'
2
+
3
+ namespace :sequel do
4
+ namespace :db do
5
+ desc 'Create databases and shards for tests'
6
+ task :create do
7
+ ENV['RACK_ENV'] ||= 'test'
8
+ Sequel::SchemaSharding.sharding_yml_path = "spec/fixtures/test_db_config.yml"
9
+ Sequel::SchemaSharding.migration_path = "spec/fixtures/db/migrate"
10
+ manager = Sequel::SchemaSharding::DatabaseManager.new
11
+ manager.create_databases
12
+ manager.create_shards
13
+ end
14
+
15
+ desc 'Create databases and shards for tests'
16
+ task :drop do
17
+ ENV['RACK_ENV'] ||= 'test'
18
+ Sequel::SchemaSharding.sharding_yml_path = "spec/fixtures/test_db_config.yml"
19
+ Sequel::SchemaSharding.migration_path = "spec/fixtures/db/migrate"
20
+ manager = Sequel::SchemaSharding::DatabaseManager.new
21
+ manager.drop_databases
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,26 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'sequel/schema-sharding/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "sequel-schema-sharding"
8
+ spec.version = Sequel::SchemaSharding::VERSION
9
+ spec.authors = ["Paul Henry", "James Hart", "Eric Saxby"]
10
+ spec.email = ["dev@wanelo.com"]
11
+ spec.description = %q{}
12
+ spec.summary = %q{Create horizontally sharded Sequel models with Postgres}
13
+ spec.homepage = "https://github.com/wanelo/sequel-schema-sharding"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files`.split($/)
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_dependency "sequel"
22
+ spec.add_dependency "pg"
23
+
24
+ spec.add_development_dependency "bundler", "~> 1.3"
25
+ spec.add_development_dependency "rake"
26
+ end
@@ -0,0 +1,13 @@
1
+ Sequel.migration do
2
+ up do
3
+ create_table(:artists) do
4
+ primary_key :id
5
+ Integer :artist_id
6
+ String :name, :null=>false
7
+ end
8
+ end
9
+
10
+ down do
11
+ drop_table(:artists)
12
+ end
13
+ end
@@ -0,0 +1,12 @@
1
+ Sequel.migration do
2
+ up do
3
+ create_table(:boofs) do
4
+ primary_key :id
5
+ String :name, :null=>false
6
+ end
7
+ end
8
+
9
+ down do
10
+ drop_table(:boofs)
11
+ end
12
+ end
@@ -0,0 +1,67 @@
1
+ test:
2
+ tables:
3
+ artists:
4
+ schema_name: sequel_logical_artists_%e_%s
5
+ logical_shards:
6
+ 1..10: shard1
7
+ 11..20: shard2
8
+ boof:
9
+ schema_name: sequel_logical_boof_%e_%s
10
+ logical_shards:
11
+ 1..10: shard1
12
+ 11..20: shard2
13
+ physical_shards:
14
+ shard1:
15
+ host: 127.0.0.1
16
+ database: sequel_test_shard1
17
+ shard2:
18
+ host: 127.0.0.1
19
+ database: sequel_test_shard2
20
+ common:
21
+ username: postgres
22
+ password: boomboomkaboom
23
+ boom:
24
+ tables:
25
+ artists:
26
+ schema_name: sequel_explosions_artists_%e_%s
27
+ logical_shards:
28
+ 1..10: shard1
29
+ 11..20: shard2
30
+ boof:
31
+ schema_name: sequel_explosions_boof_%e_%s
32
+ logical_shards:
33
+ 1..10: shard1
34
+ 11..20: shard2
35
+ physical_shards:
36
+ shard1:
37
+ host: 127.0.0.1
38
+ database: sequel_boom_shard1
39
+ shard2:
40
+ host: 127.0.0.1
41
+ database: sequel_boom_shard2
42
+ common:
43
+ username: postgres
44
+ password: boomboomkaboom
45
+ port: 5432
46
+ #boom:
47
+ # logical_shards:
48
+ # user_saves:
49
+ # 1..10: shard1
50
+ # 11..20: shard2
51
+ # product_saves:
52
+ # 1..10: shard2
53
+ # 11..20: shard3
54
+ # physical_shards:
55
+ # shard1:
56
+ # host: 127.0.0.1
57
+ # database: wanelo_saves_boom_shard1
58
+ # shard2:
59
+ # host: 127.0.0.1
60
+ # database: wanelo_saves_boom_shard2
61
+ # shard3:
62
+ # host: 127.0.0.1
63
+ # database: wanelo_saves_boom_shard3
64
+ # common:
65
+ # username: postgres
66
+ # password: boomboomkaboom
67
+ # port: 5432
@@ -0,0 +1,42 @@
1
+ require 'spec_helper'
2
+ require 'sequel/schema-sharding/configuration'
3
+
4
+ describe Sequel::SchemaSharding::Configuration do
5
+
6
+ let(:config) { Sequel::SchemaSharding::Configuration.new(:boom, 'spec/fixtures/test_db_config.yml') }
7
+
8
+ describe '#logical_shard_configs' do
9
+ it 'returns a hash representing the mapping between logical and physical shards' do
10
+ shards = config.logical_shard_configs('boof')
11
+
12
+ expect(shards.length).to eq(20)
13
+
14
+ shards.each_pair do |key, value|
15
+ if key <= 10
16
+ expect(value).to eq('shard1')
17
+ elsif key > 10
18
+ expect(value).to eq('shard2')
19
+ end
20
+ end
21
+ end
22
+ end
23
+
24
+ describe '#physical_shard_configs' do
25
+ it 'returns a hash representing the configuration for all physical shards' do
26
+ shards = config.physical_shard_configs
27
+
28
+ expect(shards.length).to eq(2)
29
+
30
+ i = 1
31
+ shards.each_pair do |key, value|
32
+ expect(value['host']).to eq('127.0.0.1')
33
+ expect(value['database']).to eq("sequel_boom_shard#{i}")
34
+ expect(value['username']).to eq('postgres')
35
+ expect(value['password']).to eq('boomboomkaboom')
36
+ expect(value['port']).to eq(5432)
37
+
38
+ i += 1
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,33 @@
1
+ require 'spec_helper'
2
+ require 'sequel/schema-sharding/connection_manager'
3
+
4
+ describe Sequel::SchemaSharding::ConnectionManager do
5
+ let(:config) { Sequel::SchemaSharding::Configuration.new('boom', 'spec/fixtures/test_db_config.yml') }
6
+
7
+ subject { Sequel::SchemaSharding::ConnectionManager.new }
8
+
9
+ before { subject.stubs(:config).returns(config) }
10
+
11
+ describe '#[]' do
12
+ it 'returns a valid connection instance for the specified physical shard' do
13
+ expect(subject['shard1']).to be_a(Sequel::Postgres::Database)
14
+ expect(subject['shard2']).to be_a(Sequel::Postgres::Database)
15
+ end
16
+ end
17
+
18
+ describe "#schema_for" do
19
+ it "returns the schema name based on env and shard number" do
20
+ subject.schema_for('boof', 'pickles', 3).should eq 'sequel_explosions_boof_pickles_3'
21
+ end
22
+ end
23
+
24
+ describe "#default_dataset_for" do
25
+ it "returns a dataset scoped to a configured schema" do
26
+ # TODO ConnectionManager is dependent on global state from Sequel::SchemaSharding.config.
27
+ # This should be deconstructed to allow for injection of a mock config for testing.
28
+ dataset = subject.default_dataset_for("artists")
29
+ expect(dataset).to be_a(Sequel::Dataset)
30
+ expect(dataset.first_source_table).to eql(:'sequel_explosions_artists_test_1__artists')
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,124 @@
1
+ require 'spec_helper'
2
+ require 'sequel/schema-sharding'
3
+
4
+ describe Sequel::SchemaSharding::DatabaseManager, type: :manager, sharded: true do
5
+
6
+ let(:config) { Sequel::SchemaSharding::Configuration.new('boom', 'spec/fixtures/test_db_config.yml') }
7
+
8
+ after do
9
+ Sequel::SchemaSharding.connection_manager.disconnect
10
+ end
11
+
12
+ around do |ex|
13
+ Sequel::SchemaSharding.stubs(:config).returns(config)
14
+
15
+ @manager = Sequel::SchemaSharding::DatabaseManager.new
16
+ @manager.send(:connection_manager).disconnect
17
+ DatabaseHelper.disconnect
18
+
19
+ DatabaseHelper.drop_db('sequel_boom_shard1')
20
+ DatabaseHelper.drop_db('sequel_boom_shard2')
21
+
22
+ ex.call
23
+ end
24
+
25
+ describe '#create_database' do
26
+ context 'database does not exist' do
27
+ it 'creates the database for the current environment' do
28
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
29
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
30
+ @manager.create_databases
31
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
32
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
33
+ end
34
+ end
35
+
36
+ context 'database exists' do
37
+
38
+ before(:each) do
39
+ @manager.create_databases
40
+ end
41
+
42
+ it 'outputs message to stderr' do
43
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
44
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
45
+ $stderr.expects(:puts).with(regexp_matches(/already exists/)).twice
46
+ @manager.create_databases
47
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
48
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
49
+ end
50
+ end
51
+ end
52
+
53
+ describe '#drop_databases' do
54
+ context 'databases exist' do
55
+ it 'drops the database for the current environment' do
56
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
57
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
58
+
59
+ @manager.create_databases
60
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
61
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
62
+
63
+ @manager.drop_databases
64
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
65
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
66
+ end
67
+ end
68
+
69
+ context 'databases dont exist' do
70
+ it 'raises an error' do
71
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
72
+ expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
73
+
74
+ $stderr.expects(:puts).with(regexp_matches(/database doesnt exist/)).times(2)
75
+
76
+ @manager.drop_databases
77
+ end
78
+ end
79
+ end
80
+
81
+ describe 'logical shards' do
82
+ before(:each) do
83
+ @manager.create_databases
84
+ end
85
+
86
+ describe '#create_shards' do
87
+ it 'creates the database structure' do
88
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
89
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
90
+ @manager.create_shards
91
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
92
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
93
+ end
94
+
95
+ context 'shards already exist' do
96
+ it 'prints that shards already exist' do
97
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
98
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
99
+ @manager.create_shards
100
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
101
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
102
+
103
+ $stderr.expects(:puts).with(regexp_matches(/already exists/)).at_least_once
104
+ @manager.create_shards
105
+ end
106
+ end
107
+ end
108
+
109
+ describe '#drop_schemas' do
110
+ context 'schemas exist' do
111
+ it 'drops the schemas' do
112
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
113
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
114
+ @manager.create_shards
115
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
116
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
117
+ @manager.drop_shards
118
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
119
+ expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
120
+ end
121
+ end
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,13 @@
1
+ require 'spec_helper'
2
+ require 'sequel/schema-sharding'
3
+
4
+ describe Sequel::SchemaSharding::Finder do
5
+
6
+ describe '.lookup' do
7
+ it 'returns an object with a valid connection and schema' do
8
+ result = Sequel::SchemaSharding::Finder.instance.lookup('boof', 60)
9
+ expect(result.connection).to be_a(Sequel::Postgres::Database)
10
+ expect(result.schema).to eq('sequel_logical_boof_test_2')
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,40 @@
1
+ require 'spec_helper'
2
+
3
+ describe Sequel::SchemaSharding, 'Model' do
4
+
5
+ let(:model) do
6
+ klass = Sequel::SchemaSharding::Model('artists')
7
+ klass.instance_eval do
8
+ set_sharded_column :artist_id
9
+ set_columns [:artist_id, :name]
10
+
11
+ def by_id(id)
12
+ shard_for(id).where(artist_id: id)
13
+ end
14
+
15
+ def self.implicit_table_name
16
+ 'artists'
17
+ end
18
+ end
19
+ klass
20
+ end
21
+
22
+ describe '#by_id' do
23
+ it 'returns a valid artist by id' do
24
+ artist = model.create(artist_id: 14, name: 'Paul')
25
+ expect(artist.id).to_not be_nil
26
+ read_back_artist = model.by_id(14).first
27
+ expect(read_back_artist).to be_a(model)
28
+ expect(read_back_artist.name).to eql('Paul')
29
+ end
30
+ end
31
+
32
+ describe '#create' do
33
+ it 'creates a valid artist' do
34
+ artist = model.create(artist_id: 234, name: 'Paul')
35
+ expect(artist).to be_a(model)
36
+ expect(artist.name).to eql('Paul')
37
+ end
38
+ end
39
+
40
+ end
@@ -0,0 +1,25 @@
1
+ require 'spec_helper'
2
+ require 'sequel/schema-sharding'
3
+
4
+ describe Sequel::SchemaSharding::Ring do
5
+
6
+ describe 'a ring of servers' do
7
+ it 'have the continuum sorted by value' do
8
+ shards = [1, 2, 3, 4, 5, 6, 7, 8]
9
+ ring = Sequel::SchemaSharding::Ring.new(shards)
10
+ previous_value = 0
11
+ ring.continuum.each do |entry|
12
+ expect(entry.value).to be > previous_value
13
+ previous_value = entry.value
14
+ end
15
+ end
16
+ end
17
+
18
+ describe '#shard_for_id' do
19
+ it 'returns a server for a given id' do
20
+ shards = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
21
+ ring = Sequel::SchemaSharding::Ring.new(shards)
22
+ expect(ring.shard_for_id(3489409)).to eq(4)
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,53 @@
1
+ ENV['RACK_ENV'] ||= 'test'
2
+
3
+ require 'bundler/setup'
4
+ Bundler.require 'test'
5
+
6
+ require 'sequel-schema-sharding'
7
+ require 'support/database_helper'
8
+ require 'mocha/api'
9
+
10
+ RSpec.configure do |config|
11
+ config.treat_symbols_as_metadata_keys_with_true_values = true
12
+ config.run_all_when_everything_filtered = true
13
+ config.filter_run :focus
14
+ config.alias_example_to :fit, focus: true
15
+ config.mock_framework = :mocha
16
+
17
+
18
+ # Run specs in random order to surface order dependencies. If you find an
19
+ # order dependency and want to debug it, you can fix the order by providing
20
+ # the seed, which is printed after each run.
21
+ # --seed 1234
22
+ config.order = 'random'
23
+
24
+ config.before :all do |ex|
25
+ Sequel::SchemaSharding.logger = Logger.new(StringIO.new)
26
+ Sequel::SchemaSharding.sharding_yml_path = "spec/fixtures/test_db_config.yml"
27
+ Sequel::SchemaSharding.migration_path = "spec/fixtures/db/migrate"
28
+ end
29
+
30
+ config.around :each do |ex|
31
+ #Sequel::SchemaSharding.config = Sequel::SchemaSharding::Configuration.new('boom', 'spec/fixtures/test_db_config.yml')
32
+
33
+ # Start transactions in each connection to the physical shards
34
+ connections = Sequel::SchemaSharding.config.physical_shard_configs.map do |shard_config|
35
+ Sequel::SchemaSharding.connection_manager[shard_config[0]]
36
+ end
37
+
38
+ start_transaction_proc = Proc.new do |connections|
39
+ if connections.length == 0
40
+ ex.run
41
+ else
42
+ connections[0].transaction do
43
+ connections.shift
44
+ start_transaction_proc.call(connections)
45
+ raise Sequel::Rollback
46
+ end
47
+ end
48
+ end
49
+
50
+ start_transaction_proc.call(connections)
51
+ end
52
+
53
+ end
@@ -0,0 +1,35 @@
1
+ require 'sequel'
2
+
3
+ class DatabaseHelper
4
+ def self.db
5
+ @db = Sequel.postgres(user: 'postgres', password: 'alkfjasdlfj', host: 'localhost')
6
+ end
7
+
8
+ def self.disconnect
9
+ db.disconnect
10
+ end
11
+
12
+ def self.drop_db(database)
13
+ if db_exists?("#{database}")
14
+ self.disconnect
15
+
16
+ db.run("DROP DATABASE #{database}")
17
+ end
18
+ end
19
+
20
+ def self.db_exists?(database)
21
+ dbs.include?(database)
22
+ end
23
+
24
+ def self.dbs
25
+ db.fetch('SELECT datname FROM pg_database WHERE datistemplate = false;').all.map { |d| d[:datname] }
26
+ end
27
+
28
+ def self.schema_exists?(database, schema)
29
+ schemas(database).include?(schema)
30
+ end
31
+
32
+ def self.schemas(database)
33
+ Sequel::SchemaSharding.connection_manager[database].fetch('select nspname from pg_catalog.pg_namespace;').all.map { |d| d[:nspname] }
34
+ end
35
+ end
metadata ADDED
@@ -0,0 +1,143 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sequel-schema-sharding
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Paul Henry
8
+ - James Hart
9
+ - Eric Saxby
10
+ autorequire:
11
+ bindir: bin
12
+ cert_chain: []
13
+ date: 2013-09-06 00:00:00.000000000 Z
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: sequel
17
+ requirement: !ruby/object:Gem::Requirement
18
+ requirements:
19
+ - - '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ requirements:
26
+ - - '>='
27
+ - !ruby/object:Gem::Version
28
+ version: '0'
29
+ - !ruby/object:Gem::Dependency
30
+ name: pg
31
+ requirement: !ruby/object:Gem::Requirement
32
+ requirements:
33
+ - - '>='
34
+ - !ruby/object:Gem::Version
35
+ version: '0'
36
+ type: :runtime
37
+ prerelease: false
38
+ version_requirements: !ruby/object:Gem::Requirement
39
+ requirements:
40
+ - - '>='
41
+ - !ruby/object:Gem::Version
42
+ version: '0'
43
+ - !ruby/object:Gem::Dependency
44
+ name: bundler
45
+ requirement: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - ~>
48
+ - !ruby/object:Gem::Version
49
+ version: '1.3'
50
+ type: :development
51
+ prerelease: false
52
+ version_requirements: !ruby/object:Gem::Requirement
53
+ requirements:
54
+ - - ~>
55
+ - !ruby/object:Gem::Version
56
+ version: '1.3'
57
+ - !ruby/object:Gem::Dependency
58
+ name: rake
59
+ requirement: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - '>='
62
+ - !ruby/object:Gem::Version
63
+ version: '0'
64
+ type: :development
65
+ prerelease: false
66
+ version_requirements: !ruby/object:Gem::Requirement
67
+ requirements:
68
+ - - '>='
69
+ - !ruby/object:Gem::Version
70
+ version: '0'
71
+ description: ''
72
+ email:
73
+ - dev@wanelo.com
74
+ executables: []
75
+ extensions: []
76
+ extra_rdoc_files: []
77
+ files:
78
+ - .gitignore
79
+ - .rspec
80
+ - .travis.yml
81
+ - Gemfile
82
+ - LICENSE.txt
83
+ - README.md
84
+ - Rakefile
85
+ - lib/sequel-schema-sharding.rb
86
+ - lib/sequel/schema-sharding.rb
87
+ - lib/sequel/schema-sharding/configuration.rb
88
+ - lib/sequel/schema-sharding/connection_manager.rb
89
+ - lib/sequel/schema-sharding/database_manager.rb
90
+ - lib/sequel/schema-sharding/finder.rb
91
+ - lib/sequel/schema-sharding/model.rb
92
+ - lib/sequel/schema-sharding/ring.rb
93
+ - lib/sequel/schema-sharding/sequel_ext.rb
94
+ - lib/sequel/schema-sharding/version.rb
95
+ - lib/sequel/tasks/test.rake
96
+ - sequel-schema-sharding.gemspec
97
+ - spec/fixtures/db/migrate/artists/001_create_test_table.rb
98
+ - spec/fixtures/db/migrate/boof/001_create_boof_table.rb
99
+ - spec/fixtures/test_db_config.yml
100
+ - spec/schema-sharding/configuration_spec.rb
101
+ - spec/schema-sharding/connection_manager_spec.rb
102
+ - spec/schema-sharding/database_manager_spec.rb
103
+ - spec/schema-sharding/finder_spec.rb
104
+ - spec/schema-sharding/model_spec.rb
105
+ - spec/schema-sharding/ring_spec.rb
106
+ - spec/spec_helper.rb
107
+ - spec/support/database_helper.rb
108
+ homepage: https://github.com/wanelo/sequel-schema-sharding
109
+ licenses:
110
+ - MIT
111
+ metadata: {}
112
+ post_install_message:
113
+ rdoc_options: []
114
+ require_paths:
115
+ - lib
116
+ required_ruby_version: !ruby/object:Gem::Requirement
117
+ requirements:
118
+ - - '>='
119
+ - !ruby/object:Gem::Version
120
+ version: '0'
121
+ required_rubygems_version: !ruby/object:Gem::Requirement
122
+ requirements:
123
+ - - '>='
124
+ - !ruby/object:Gem::Version
125
+ version: '0'
126
+ requirements: []
127
+ rubyforge_project:
128
+ rubygems_version: 2.0.7
129
+ signing_key:
130
+ specification_version: 4
131
+ summary: Create horizontally sharded Sequel models with Postgres
132
+ test_files:
133
+ - spec/fixtures/db/migrate/artists/001_create_test_table.rb
134
+ - spec/fixtures/db/migrate/boof/001_create_boof_table.rb
135
+ - spec/fixtures/test_db_config.yml
136
+ - spec/schema-sharding/configuration_spec.rb
137
+ - spec/schema-sharding/connection_manager_spec.rb
138
+ - spec/schema-sharding/database_manager_spec.rb
139
+ - spec/schema-sharding/finder_spec.rb
140
+ - spec/schema-sharding/model_spec.rb
141
+ - spec/schema-sharding/ring_spec.rb
142
+ - spec/spec_helper.rb
143
+ - spec/support/database_helper.rb