sequel-schema-sharding 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.rspec +2 -0
- data/.travis.yml +8 -0
- data/Gemfile +10 -0
- data/LICENSE.txt +22 -0
- data/README.md +33 -0
- data/Rakefile +2 -0
- data/lib/sequel-schema-sharding.rb +1 -0
- data/lib/sequel/schema-sharding.rb +54 -0
- data/lib/sequel/schema-sharding/configuration.rb +54 -0
- data/lib/sequel/schema-sharding/connection_manager.rb +48 -0
- data/lib/sequel/schema-sharding/database_manager.rb +105 -0
- data/lib/sequel/schema-sharding/finder.rb +43 -0
- data/lib/sequel/schema-sharding/model.rb +91 -0
- data/lib/sequel/schema-sharding/ring.rb +118 -0
- data/lib/sequel/schema-sharding/sequel_ext.rb +14 -0
- data/lib/sequel/schema-sharding/version.rb +5 -0
- data/lib/sequel/tasks/test.rake +24 -0
- data/sequel-schema-sharding.gemspec +26 -0
- data/spec/fixtures/db/migrate/artists/001_create_test_table.rb +13 -0
- data/spec/fixtures/db/migrate/boof/001_create_boof_table.rb +12 -0
- data/spec/fixtures/test_db_config.yml +67 -0
- data/spec/schema-sharding/configuration_spec.rb +42 -0
- data/spec/schema-sharding/connection_manager_spec.rb +33 -0
- data/spec/schema-sharding/database_manager_spec.rb +124 -0
- data/spec/schema-sharding/finder_spec.rb +13 -0
- data/spec/schema-sharding/model_spec.rb +40 -0
- data/spec/schema-sharding/ring_spec.rb +25 -0
- data/spec/spec_helper.rb +53 -0
- data/spec/support/database_helper.rb +35 -0
- metadata +143 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 9a83484015fdcd8dc4799e752d6d82ee94498738
|
4
|
+
data.tar.gz: 2e44e0d65a6c598371035daddc1724022b3c893b
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: bac9d7a0ed39dbe558e2f0bd6422bda34f68676662ba5283722340c1ffbf784f6e8c433ca6b118efe41c8a0e66ed123c8bfd2f482597ace1ab750a2b7d2fcbd1
|
7
|
+
data.tar.gz: 724cae47d487ee6e63a8e969ae8ed9f1d380ffaf61b3877074578a9704565fd0290ffd27550904afc9ad3949aba536b4fa69d5209670a39b13f21ae2e0fd874c
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013 TODO: Write your name
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
Sequel::SchemaSharding
|
2
|
+
================
|
3
|
+
|
4
|
+
[](https://travis-ci.org/wanelo/sequel-sharding)
|
5
|
+
|
6
|
+
Horizontally shard postgres with the Sequel gem. This gem allows you to configure mappings between logical and
|
7
|
+
physical shards, and pool connections between logical shards on the same physical server.
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
Add this line to your application's Gemfile:
|
12
|
+
|
13
|
+
gem 'sequel-sharding'
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install sequel-sharding
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
TODO :)
|
26
|
+
|
27
|
+
## Contributing
|
28
|
+
|
29
|
+
1. Fork it
|
30
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
31
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
32
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
33
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require 'sequel/schema-sharding'
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'sequel/schema-sharding/version'
|
2
|
+
require 'sequel/schema-sharding/configuration'
|
3
|
+
require 'sequel/schema-sharding/connection_manager'
|
4
|
+
require 'sequel/schema-sharding/database_manager'
|
5
|
+
require 'sequel/schema-sharding/ring'
|
6
|
+
require 'sequel/schema-sharding/finder'
|
7
|
+
require 'sequel/schema-sharding/sequel_ext'
|
8
|
+
require 'sequel/schema-sharding/model'
|
9
|
+
require 'logger'
|
10
|
+
|
11
|
+
module Sequel
|
12
|
+
module SchemaSharding
|
13
|
+
def self.config
|
14
|
+
@config ||= Sequel::SchemaSharding::Configuration.new(ENV['RACK_ENV'], sharding_yml_path)
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.config=(config)
|
18
|
+
@config = config
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.logger
|
22
|
+
@logger ||= Logger.new($stdout)
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.logger=(logger)
|
26
|
+
@logger = logger
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.connection_manager
|
30
|
+
@connection_manager ||= ConnectionManager.new
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.connection_manager=(connection_manager)
|
34
|
+
@connection_manager = connection_manager
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.sharding_yml_path
|
38
|
+
@sharding_yml_path ||= File.expand_path('../../../config/sharding.yml', __FILE__)
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.sharding_yml_path=(path)
|
42
|
+
@sharding_yml_path = path
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.migration_path
|
46
|
+
@migration_path || raise('You must set the migration path.')
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.migration_path=(path)
|
50
|
+
@migration_path = path
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
module Sequel
|
4
|
+
module SchemaSharding
|
5
|
+
class Configuration
|
6
|
+
attr_reader :env, :yaml_path
|
7
|
+
|
8
|
+
def initialize(env, yaml_path)
|
9
|
+
@env = env
|
10
|
+
@yaml_path = yaml_path
|
11
|
+
end
|
12
|
+
|
13
|
+
def physical_shard_configs
|
14
|
+
@physical_shard_configs ||= config['physical_shards'].inject({}) do |hash, value|
|
15
|
+
hash[value[0]] = config['common'].merge(value[1])
|
16
|
+
hash
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def logical_shard_configs(table_name)
|
21
|
+
table_name = table_name.to_s
|
22
|
+
@logical_shard_table_configs ||= {}
|
23
|
+
@logical_shard_table_configs[table_name] ||= begin
|
24
|
+
table_configs = config['tables'][table_name]
|
25
|
+
raise "Unknown table #{table_name} in configuration" if table_configs.nil?
|
26
|
+
table_configs['logical_shards'].inject({}) do |hash, value|
|
27
|
+
eval(value[0]).each do |i|
|
28
|
+
hash[i] = value[1]
|
29
|
+
end
|
30
|
+
hash
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def table_names
|
36
|
+
config['tables'].keys
|
37
|
+
end
|
38
|
+
|
39
|
+
def schema_name(table_name)
|
40
|
+
config['tables'][table_name.to_s]['schema_name']
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def config
|
46
|
+
yaml[env.to_s]
|
47
|
+
end
|
48
|
+
|
49
|
+
def yaml
|
50
|
+
@raw_yaml ||= YAML.load_file(yaml_path)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
|
3
|
+
module Sequel
|
4
|
+
module SchemaSharding
|
5
|
+
class ConnectionManager
|
6
|
+
attr_reader :connections
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@connections = {}
|
10
|
+
end
|
11
|
+
|
12
|
+
def [](name)
|
13
|
+
config = db_config_for(name)
|
14
|
+
@connections[name.to_s] ||= Sequel.postgres(:user => config['username'],
|
15
|
+
:password => config['password'],
|
16
|
+
:host => config['host'],
|
17
|
+
:database => config['database'])
|
18
|
+
end
|
19
|
+
|
20
|
+
def disconnect
|
21
|
+
@connections.each_value do |conn|
|
22
|
+
conn.disconnect
|
23
|
+
end
|
24
|
+
@connections = {}
|
25
|
+
end
|
26
|
+
|
27
|
+
def schema_for(table_name, environment, shard_number)
|
28
|
+
config.schema_name(table_name).gsub('%e', environment).gsub('%s', shard_number.to_s)
|
29
|
+
end
|
30
|
+
|
31
|
+
def default_dataset_for(table_name)
|
32
|
+
shard_number = config.logical_shard_configs(table_name).keys.first
|
33
|
+
shard_name = config.logical_shard_configs(table_name)[shard_number]
|
34
|
+
self[shard_name][:"#{schema_for(table_name, ENV['RACK_ENV'], shard_number)}__#{table_name}"]
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def db_config_for(name)
|
40
|
+
config.physical_shard_configs[name]
|
41
|
+
end
|
42
|
+
|
43
|
+
def config
|
44
|
+
Sequel::SchemaSharding.config
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
require 'sequel/schema-sharding/connection_manager'
|
3
|
+
|
4
|
+
Sequel.extension :migration
|
5
|
+
|
6
|
+
module Sequel
|
7
|
+
module SchemaSharding
|
8
|
+
class DatabaseManager
|
9
|
+
def create_databases
|
10
|
+
config.physical_shard_configs.each_pair do |name, config|
|
11
|
+
begin
|
12
|
+
# Need to create connection manually with specifying a database in order to create the database
|
13
|
+
connection = Sequel.postgres(:user => config['username'],
|
14
|
+
:password => config['password'],
|
15
|
+
:host => config['host'])
|
16
|
+
|
17
|
+
Sequel::SchemaSharding.logger.info "Creating #{config['database']}.."
|
18
|
+
|
19
|
+
connection.run("CREATE DATABASE #{config['database']}")
|
20
|
+
rescue Sequel::DatabaseError => e
|
21
|
+
if e.message.include?('already exists')
|
22
|
+
$stderr.puts "#{config['database']} database already exists"
|
23
|
+
else
|
24
|
+
raise e
|
25
|
+
end
|
26
|
+
ensure
|
27
|
+
connection.disconnect
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def drop_databases
|
33
|
+
connection_manager.disconnect
|
34
|
+
config.physical_shard_configs.each_pair do |name, config|
|
35
|
+
# Need to create connection manually with specifying a database in order to create the database
|
36
|
+
begin
|
37
|
+
connection = Sequel.postgres(:user => config['username'],
|
38
|
+
:password => config['password'],
|
39
|
+
:host => config['host'])
|
40
|
+
|
41
|
+
Sequel::SchemaSharding.logger.info "Dropping #{config['database']}.."
|
42
|
+
connection.run("DROP DATABASE #{config['database']}")
|
43
|
+
rescue Sequel::DatabaseError => e
|
44
|
+
if e.message.include?('does not exist')
|
45
|
+
$stderr.puts "#{config['database']} database doesnt exist"
|
46
|
+
else
|
47
|
+
raise e
|
48
|
+
end
|
49
|
+
ensure
|
50
|
+
connection.disconnect
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def create_shards
|
56
|
+
config.table_names.each do |table_name|
|
57
|
+
config.logical_shard_configs(table_name).each_pair do |shard_number, physical_shard|
|
58
|
+
schema_name = connection_manager.schema_for(table_name, env, shard_number)
|
59
|
+
Sequel::SchemaSharding.logger.info "Creating schema #{schema_name} on #{physical_shard}.."
|
60
|
+
connection = connection_manager[physical_shard]
|
61
|
+
|
62
|
+
begin
|
63
|
+
connection.run("CREATE SCHEMA #{schema_name}")
|
64
|
+
rescue Sequel::DatabaseError => e
|
65
|
+
if e.message.include?('already exists')
|
66
|
+
$stderr.puts "#{schema_name} schema already exists"
|
67
|
+
else
|
68
|
+
raise e
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
connection.run("SET search_path TO #{schema_name}")
|
73
|
+
|
74
|
+
Sequel::Migrator.run(connection, Sequel::SchemaSharding.migration_path + "/#{table_name}", :use_transactions => true)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def drop_shards
|
80
|
+
config.table_names.each do |table_name|
|
81
|
+
config.logical_shard_configs(table_name).each_pair do |shard_number, physical_shard|
|
82
|
+
schema_name = connection_manager.schema_for(table_name, env, shard_number)
|
83
|
+
Sequel::SchemaSharding.logger.info "Dropping schema #{schema_name} on #{physical_shard}.."
|
84
|
+
connection = connection_manager[physical_shard]
|
85
|
+
connection.run("DROP SCHEMA #{schema_name} CASCADE")
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
private
|
91
|
+
|
92
|
+
def env
|
93
|
+
config.env
|
94
|
+
end
|
95
|
+
|
96
|
+
def config
|
97
|
+
Sequel::SchemaSharding.config
|
98
|
+
end
|
99
|
+
|
100
|
+
def connection_manager
|
101
|
+
Sequel::SchemaSharding.connection_manager
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
|
3
|
+
module Sequel
|
4
|
+
module SchemaSharding
|
5
|
+
class Finder
|
6
|
+
class Result
|
7
|
+
attr_reader :connection, :schema
|
8
|
+
|
9
|
+
def initialize(connection, schema)
|
10
|
+
@connection = connection
|
11
|
+
@schema = schema
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
include ::Singleton
|
16
|
+
|
17
|
+
def lookup(table_name, id)
|
18
|
+
shard_number = shard_for_id(table_name, id)
|
19
|
+
physical_shard = config.logical_shard_configs(table_name)[shard_number]
|
20
|
+
|
21
|
+
conn = Sequel::SchemaSharding.connection_manager[physical_shard]
|
22
|
+
schema = Sequel::SchemaSharding.connection_manager.schema_for(table_name, config.env, shard_number)
|
23
|
+
|
24
|
+
Result.new(conn, schema)
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def shard_for_id(table_name, id)
|
30
|
+
ring(table_name).shard_for_id(id)
|
31
|
+
end
|
32
|
+
|
33
|
+
def ring(table_name)
|
34
|
+
@rings ||= {}
|
35
|
+
@rings[table_name] ||= Sequel::SchemaSharding::Ring.new(config.logical_shard_configs(table_name).keys)
|
36
|
+
end
|
37
|
+
|
38
|
+
def config
|
39
|
+
Sequel::SchemaSharding.config
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
|
3
|
+
module Sequel
|
4
|
+
module SchemaSharding
|
5
|
+
# Extensions to the Sequel model to allow logical/physical shards. Actual table models should
|
6
|
+
# inherit this class like so:
|
7
|
+
#
|
8
|
+
# class Cat < Sequel::SchemaSharding::Model
|
9
|
+
# set_columns [:cat_id, :fur, :tongue, :whiskers] # Columns in the database need to be predefined.
|
10
|
+
# set_sharded_column :cat_id # Define the shard column
|
11
|
+
#
|
12
|
+
# def self.by_cat_id(id)
|
13
|
+
# # You should always call shard_for in finders to select the correct connection.
|
14
|
+
# shard_for(id).where(cat_id: id)
|
15
|
+
# end
|
16
|
+
# end
|
17
|
+
|
18
|
+
def self.Model(source)
|
19
|
+
klass = Sequel::Model(Sequel::SchemaSharding.connection_manager.default_dataset_for(source))
|
20
|
+
|
21
|
+
klass.include(SchemaSharding::ShardedModel)
|
22
|
+
|
23
|
+
klass
|
24
|
+
end
|
25
|
+
|
26
|
+
module ShardedModel
|
27
|
+
|
28
|
+
def self.included(base)
|
29
|
+
base.extend(ClassMethods)
|
30
|
+
end
|
31
|
+
|
32
|
+
module ClassMethods
|
33
|
+
|
34
|
+
#protected
|
35
|
+
|
36
|
+
# Set the column on which the current model is sharded. This is used when saving, inserting and finding
|
37
|
+
# to decide which connection to use.
|
38
|
+
def set_sharded_column(column)
|
39
|
+
@sharded_column = column
|
40
|
+
end
|
41
|
+
|
42
|
+
# Accessor for the sharded_columns
|
43
|
+
def sharded_column
|
44
|
+
@sharded_column
|
45
|
+
end
|
46
|
+
|
47
|
+
# Return a valid Sequel::Dataset that is tied to the shard table and connection for the id and will load values
|
48
|
+
# run by the query into the model.
|
49
|
+
def shard_for(id)
|
50
|
+
result = self.result_for(id)
|
51
|
+
ds = result.connection[schema_and_table(result)]
|
52
|
+
ds.row_proc = self
|
53
|
+
dataset_method_modules.each { |m| ds.instance_eval { extend(m) } }
|
54
|
+
ds.model = self
|
55
|
+
ds
|
56
|
+
end
|
57
|
+
|
58
|
+
# The result of a lookup for the given id. See Sequel::SchemaSharding::Finder::Result
|
59
|
+
def result_for(id)
|
60
|
+
Sequel::SchemaSharding::Finder.instance.lookup(self.implicit_table_name, id)
|
61
|
+
end
|
62
|
+
|
63
|
+
# Construct the schema and table for use in a dataset.
|
64
|
+
def schema_and_table(result)
|
65
|
+
:"#{result.schema}__#{self.implicit_table_name}"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# The database connection that has the logical shard.
|
70
|
+
def db
|
71
|
+
@db ||= finder_result.connection
|
72
|
+
end
|
73
|
+
|
74
|
+
# Wrapper for performing the sharding lookup based on the sharded column.
|
75
|
+
def finder_result
|
76
|
+
@result ||= self.class.result_for(self.send(self.class.sharded_column))
|
77
|
+
end
|
78
|
+
|
79
|
+
# Dataset instance based on the sharded column.
|
80
|
+
def this_server
|
81
|
+
@this_server ||= db[self.class.schema_and_table(finder_result)]
|
82
|
+
end
|
83
|
+
|
84
|
+
# Overriden to not use @dataset value from the Sequel::Model. Used internally only.
|
85
|
+
def _insert_dataset
|
86
|
+
this_server
|
87
|
+
end
|
88
|
+
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
require 'digest/sha1'
|
2
|
+
require 'zlib'
|
3
|
+
|
4
|
+
module Sequel
|
5
|
+
module SchemaSharding
|
6
|
+
class Ring
|
7
|
+
POINTS_PER_SERVER = 1
|
8
|
+
|
9
|
+
attr_accessor :shards, :continuum
|
10
|
+
|
11
|
+
def initialize(shards)
|
12
|
+
@shards = shards
|
13
|
+
@continuum = nil
|
14
|
+
if shards.size > 1
|
15
|
+
continuum = []
|
16
|
+
shards.each do |shard|
|
17
|
+
hash = Digest::SHA1.hexdigest("#{shard}")
|
18
|
+
value = Integer("0x#{hash[0..7]}")
|
19
|
+
continuum << Entry.new(value, shard)
|
20
|
+
end
|
21
|
+
@continuum = continuum.sort { |a, b| a.value <=> b.value }
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def shard_for_id(id)
|
26
|
+
if @continuum
|
27
|
+
hkey = hash_for(id)
|
28
|
+
entryidx = binary_search(@continuum, hkey)
|
29
|
+
return @continuum[entryidx].server
|
30
|
+
else
|
31
|
+
server = @servers.first
|
32
|
+
return server if server
|
33
|
+
end
|
34
|
+
|
35
|
+
raise StandardError, "No server available"
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def hash_for(key)
|
41
|
+
Zlib.crc32(key.to_s)
|
42
|
+
end
|
43
|
+
|
44
|
+
def entry_count
|
45
|
+
((shards.size * POINTS_PER_SERVER)).floor
|
46
|
+
end
|
47
|
+
|
48
|
+
# Native extension to perform the binary search within the continuum
|
49
|
+
# space. Fallback to a pure Ruby version if the compilation doesn't work.
|
50
|
+
# optional for performance and only necessary if you are using multiple
|
51
|
+
# memcached servers.
|
52
|
+
begin
|
53
|
+
require 'inline'
|
54
|
+
inline do |builder|
|
55
|
+
builder.c <<-EOM
|
56
|
+
int binary_search(VALUE ary, unsigned int r) {
|
57
|
+
long upper = RARRAY_LEN(ary) - 1;
|
58
|
+
long lower = 0;
|
59
|
+
long idx = 0;
|
60
|
+
ID value = rb_intern("value");
|
61
|
+
VALUE continuumValue;
|
62
|
+
unsigned int l;
|
63
|
+
|
64
|
+
while (lower <= upper) {
|
65
|
+
idx = (lower + upper) / 2;
|
66
|
+
|
67
|
+
continuumValue = rb_funcall(RARRAY_PTR(ary)[idx], value, 0);
|
68
|
+
l = NUM2UINT(continuumValue);
|
69
|
+
if (l == r) {
|
70
|
+
return idx;
|
71
|
+
}
|
72
|
+
else if (l > r) {
|
73
|
+
upper = idx - 1;
|
74
|
+
}
|
75
|
+
else {
|
76
|
+
lower = idx + 1;
|
77
|
+
}
|
78
|
+
}
|
79
|
+
return upper;
|
80
|
+
}
|
81
|
+
EOM
|
82
|
+
end
|
83
|
+
rescue LoadError
|
84
|
+
# Find the closest index in the Ring with value <= the given value
|
85
|
+
def binary_search(ary, value)
|
86
|
+
upper = ary.size - 1
|
87
|
+
lower = 0
|
88
|
+
idx = 0
|
89
|
+
|
90
|
+
while (lower <= upper) do
|
91
|
+
idx = (lower + upper) / 2
|
92
|
+
comp = ary[idx].value <=> value
|
93
|
+
|
94
|
+
if comp == 0
|
95
|
+
return idx
|
96
|
+
elsif comp > 0
|
97
|
+
upper = idx - 1
|
98
|
+
else
|
99
|
+
lower = idx + 1
|
100
|
+
end
|
101
|
+
end
|
102
|
+
return upper
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
class Entry
|
107
|
+
attr_reader :value
|
108
|
+
attr_reader :server
|
109
|
+
|
110
|
+
def initialize(val, srv)
|
111
|
+
@value = val
|
112
|
+
@server = srv
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'sequel/schema-sharding'
|
2
|
+
|
3
|
+
namespace :sequel do
|
4
|
+
namespace :db do
|
5
|
+
desc 'Create databases and shards for tests'
|
6
|
+
task :create do
|
7
|
+
ENV['RACK_ENV'] ||= 'test'
|
8
|
+
Sequel::SchemaSharding.sharding_yml_path = "spec/fixtures/test_db_config.yml"
|
9
|
+
Sequel::SchemaSharding.migration_path = "spec/fixtures/db/migrate"
|
10
|
+
manager = Sequel::SchemaSharding::DatabaseManager.new
|
11
|
+
manager.create_databases
|
12
|
+
manager.create_shards
|
13
|
+
end
|
14
|
+
|
15
|
+
desc 'Create databases and shards for tests'
|
16
|
+
task :drop do
|
17
|
+
ENV['RACK_ENV'] ||= 'test'
|
18
|
+
Sequel::SchemaSharding.sharding_yml_path = "spec/fixtures/test_db_config.yml"
|
19
|
+
Sequel::SchemaSharding.migration_path = "spec/fixtures/db/migrate"
|
20
|
+
manager = Sequel::SchemaSharding::DatabaseManager.new
|
21
|
+
manager.drop_databases
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'sequel/schema-sharding/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "sequel-schema-sharding"
|
8
|
+
spec.version = Sequel::SchemaSharding::VERSION
|
9
|
+
spec.authors = ["Paul Henry", "James Hart", "Eric Saxby"]
|
10
|
+
spec.email = ["dev@wanelo.com"]
|
11
|
+
spec.description = %q{}
|
12
|
+
spec.summary = %q{Create horizontally sharded Sequel models with Postgres}
|
13
|
+
spec.homepage = "https://github.com/wanelo/sequel-schema-sharding"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_dependency "sequel"
|
22
|
+
spec.add_dependency "pg"
|
23
|
+
|
24
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
25
|
+
spec.add_development_dependency "rake"
|
26
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
test:
|
2
|
+
tables:
|
3
|
+
artists:
|
4
|
+
schema_name: sequel_logical_artists_%e_%s
|
5
|
+
logical_shards:
|
6
|
+
1..10: shard1
|
7
|
+
11..20: shard2
|
8
|
+
boof:
|
9
|
+
schema_name: sequel_logical_boof_%e_%s
|
10
|
+
logical_shards:
|
11
|
+
1..10: shard1
|
12
|
+
11..20: shard2
|
13
|
+
physical_shards:
|
14
|
+
shard1:
|
15
|
+
host: 127.0.0.1
|
16
|
+
database: sequel_test_shard1
|
17
|
+
shard2:
|
18
|
+
host: 127.0.0.1
|
19
|
+
database: sequel_test_shard2
|
20
|
+
common:
|
21
|
+
username: postgres
|
22
|
+
password: boomboomkaboom
|
23
|
+
boom:
|
24
|
+
tables:
|
25
|
+
artists:
|
26
|
+
schema_name: sequel_explosions_artists_%e_%s
|
27
|
+
logical_shards:
|
28
|
+
1..10: shard1
|
29
|
+
11..20: shard2
|
30
|
+
boof:
|
31
|
+
schema_name: sequel_explosions_boof_%e_%s
|
32
|
+
logical_shards:
|
33
|
+
1..10: shard1
|
34
|
+
11..20: shard2
|
35
|
+
physical_shards:
|
36
|
+
shard1:
|
37
|
+
host: 127.0.0.1
|
38
|
+
database: sequel_boom_shard1
|
39
|
+
shard2:
|
40
|
+
host: 127.0.0.1
|
41
|
+
database: sequel_boom_shard2
|
42
|
+
common:
|
43
|
+
username: postgres
|
44
|
+
password: boomboomkaboom
|
45
|
+
port: 5432
|
46
|
+
#boom:
|
47
|
+
# logical_shards:
|
48
|
+
# user_saves:
|
49
|
+
# 1..10: shard1
|
50
|
+
# 11..20: shard2
|
51
|
+
# product_saves:
|
52
|
+
# 1..10: shard2
|
53
|
+
# 11..20: shard3
|
54
|
+
# physical_shards:
|
55
|
+
# shard1:
|
56
|
+
# host: 127.0.0.1
|
57
|
+
# database: wanelo_saves_boom_shard1
|
58
|
+
# shard2:
|
59
|
+
# host: 127.0.0.1
|
60
|
+
# database: wanelo_saves_boom_shard2
|
61
|
+
# shard3:
|
62
|
+
# host: 127.0.0.1
|
63
|
+
# database: wanelo_saves_boom_shard3
|
64
|
+
# common:
|
65
|
+
# username: postgres
|
66
|
+
# password: boomboomkaboom
|
67
|
+
# port: 5432
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sequel/schema-sharding/configuration'
|
3
|
+
|
4
|
+
describe Sequel::SchemaSharding::Configuration do
|
5
|
+
|
6
|
+
let(:config) { Sequel::SchemaSharding::Configuration.new(:boom, 'spec/fixtures/test_db_config.yml') }
|
7
|
+
|
8
|
+
describe '#logical_shard_configs' do
|
9
|
+
it 'returns a hash representing the mapping between logical and physical shards' do
|
10
|
+
shards = config.logical_shard_configs('boof')
|
11
|
+
|
12
|
+
expect(shards.length).to eq(20)
|
13
|
+
|
14
|
+
shards.each_pair do |key, value|
|
15
|
+
if key <= 10
|
16
|
+
expect(value).to eq('shard1')
|
17
|
+
elsif key > 10
|
18
|
+
expect(value).to eq('shard2')
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
describe '#physical_shard_configs' do
|
25
|
+
it 'returns a hash representing the configuration for all physical shards' do
|
26
|
+
shards = config.physical_shard_configs
|
27
|
+
|
28
|
+
expect(shards.length).to eq(2)
|
29
|
+
|
30
|
+
i = 1
|
31
|
+
shards.each_pair do |key, value|
|
32
|
+
expect(value['host']).to eq('127.0.0.1')
|
33
|
+
expect(value['database']).to eq("sequel_boom_shard#{i}")
|
34
|
+
expect(value['username']).to eq('postgres')
|
35
|
+
expect(value['password']).to eq('boomboomkaboom')
|
36
|
+
expect(value['port']).to eq(5432)
|
37
|
+
|
38
|
+
i += 1
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sequel/schema-sharding/connection_manager'
|
3
|
+
|
4
|
+
describe Sequel::SchemaSharding::ConnectionManager do
|
5
|
+
let(:config) { Sequel::SchemaSharding::Configuration.new('boom', 'spec/fixtures/test_db_config.yml') }
|
6
|
+
|
7
|
+
subject { Sequel::SchemaSharding::ConnectionManager.new }
|
8
|
+
|
9
|
+
before { subject.stubs(:config).returns(config) }
|
10
|
+
|
11
|
+
describe '#[]' do
|
12
|
+
it 'returns a valid connection instance for the specified physical shard' do
|
13
|
+
expect(subject['shard1']).to be_a(Sequel::Postgres::Database)
|
14
|
+
expect(subject['shard2']).to be_a(Sequel::Postgres::Database)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
describe "#schema_for" do
|
19
|
+
it "returns the schema name based on env and shard number" do
|
20
|
+
subject.schema_for('boof', 'pickles', 3).should eq 'sequel_explosions_boof_pickles_3'
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
describe "#default_dataset_for" do
|
25
|
+
it "returns a dataset scoped to a configured schema" do
|
26
|
+
# TODO ConnectionManager is dependent on global state from Sequel::SchemaSharding.config.
|
27
|
+
# This should be deconstructed to allow for injection of a mock config for testing.
|
28
|
+
dataset = subject.default_dataset_for("artists")
|
29
|
+
expect(dataset).to be_a(Sequel::Dataset)
|
30
|
+
expect(dataset.first_source_table).to eql(:'sequel_explosions_artists_test_1__artists')
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,124 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sequel/schema-sharding'
|
3
|
+
|
4
|
+
describe Sequel::SchemaSharding::DatabaseManager, type: :manager, sharded: true do
|
5
|
+
|
6
|
+
let(:config) { Sequel::SchemaSharding::Configuration.new('boom', 'spec/fixtures/test_db_config.yml') }
|
7
|
+
|
8
|
+
after do
|
9
|
+
Sequel::SchemaSharding.connection_manager.disconnect
|
10
|
+
end
|
11
|
+
|
12
|
+
around do |ex|
|
13
|
+
Sequel::SchemaSharding.stubs(:config).returns(config)
|
14
|
+
|
15
|
+
@manager = Sequel::SchemaSharding::DatabaseManager.new
|
16
|
+
@manager.send(:connection_manager).disconnect
|
17
|
+
DatabaseHelper.disconnect
|
18
|
+
|
19
|
+
DatabaseHelper.drop_db('sequel_boom_shard1')
|
20
|
+
DatabaseHelper.drop_db('sequel_boom_shard2')
|
21
|
+
|
22
|
+
ex.call
|
23
|
+
end
|
24
|
+
|
25
|
+
describe '#create_database' do
|
26
|
+
context 'database does not exist' do
|
27
|
+
it 'creates the database for the current environment' do
|
28
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
|
29
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
|
30
|
+
@manager.create_databases
|
31
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
|
32
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
context 'database exists' do
|
37
|
+
|
38
|
+
before(:each) do
|
39
|
+
@manager.create_databases
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'outputs message to stderr' do
|
43
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
|
44
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
|
45
|
+
$stderr.expects(:puts).with(regexp_matches(/already exists/)).twice
|
46
|
+
@manager.create_databases
|
47
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
|
48
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
describe '#drop_databases' do
|
54
|
+
context 'databases exist' do
|
55
|
+
it 'drops the database for the current environment' do
|
56
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
|
57
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
|
58
|
+
|
59
|
+
@manager.create_databases
|
60
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_true
|
61
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_true
|
62
|
+
|
63
|
+
@manager.drop_databases
|
64
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
|
65
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
context 'databases dont exist' do
|
70
|
+
it 'raises an error' do
|
71
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard1')).to be_false
|
72
|
+
expect(DatabaseHelper.db_exists?('sequel_boom_shard2')).to be_false
|
73
|
+
|
74
|
+
$stderr.expects(:puts).with(regexp_matches(/database doesnt exist/)).times(2)
|
75
|
+
|
76
|
+
@manager.drop_databases
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
describe 'logical shards' do
|
82
|
+
before(:each) do
|
83
|
+
@manager.create_databases
|
84
|
+
end
|
85
|
+
|
86
|
+
describe '#create_shards' do
|
87
|
+
it 'creates the database structure' do
|
88
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
|
89
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
|
90
|
+
@manager.create_shards
|
91
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
|
92
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
|
93
|
+
end
|
94
|
+
|
95
|
+
context 'shards already exist' do
|
96
|
+
it 'prints that shards already exist' do
|
97
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
|
98
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
|
99
|
+
@manager.create_shards
|
100
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
|
101
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
|
102
|
+
|
103
|
+
$stderr.expects(:puts).with(regexp_matches(/already exists/)).at_least_once
|
104
|
+
@manager.create_shards
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
describe '#drop_schemas' do
|
110
|
+
context 'schemas exist' do
|
111
|
+
it 'drops the schemas' do
|
112
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
|
113
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
|
114
|
+
@manager.create_shards
|
115
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_true
|
116
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_true
|
117
|
+
@manager.drop_shards
|
118
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_boof_boom_1')).to be_false
|
119
|
+
expect(DatabaseHelper.schema_exists?('shard1', 'sequel_explosions_artists_boom_1')).to be_false
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sequel/schema-sharding'
|
3
|
+
|
4
|
+
describe Sequel::SchemaSharding::Finder do
|
5
|
+
|
6
|
+
describe '.lookup' do
|
7
|
+
it 'returns an object with a valid connection and schema' do
|
8
|
+
result = Sequel::SchemaSharding::Finder.instance.lookup('boof', 60)
|
9
|
+
expect(result.connection).to be_a(Sequel::Postgres::Database)
|
10
|
+
expect(result.schema).to eq('sequel_logical_boof_test_2')
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Sequel::SchemaSharding, 'Model' do
|
4
|
+
|
5
|
+
let(:model) do
|
6
|
+
klass = Sequel::SchemaSharding::Model('artists')
|
7
|
+
klass.instance_eval do
|
8
|
+
set_sharded_column :artist_id
|
9
|
+
set_columns [:artist_id, :name]
|
10
|
+
|
11
|
+
def by_id(id)
|
12
|
+
shard_for(id).where(artist_id: id)
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.implicit_table_name
|
16
|
+
'artists'
|
17
|
+
end
|
18
|
+
end
|
19
|
+
klass
|
20
|
+
end
|
21
|
+
|
22
|
+
describe '#by_id' do
|
23
|
+
it 'returns a valid artist by id' do
|
24
|
+
artist = model.create(artist_id: 14, name: 'Paul')
|
25
|
+
expect(artist.id).to_not be_nil
|
26
|
+
read_back_artist = model.by_id(14).first
|
27
|
+
expect(read_back_artist).to be_a(model)
|
28
|
+
expect(read_back_artist.name).to eql('Paul')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
describe '#create' do
|
33
|
+
it 'creates a valid artist' do
|
34
|
+
artist = model.create(artist_id: 234, name: 'Paul')
|
35
|
+
expect(artist).to be_a(model)
|
36
|
+
expect(artist.name).to eql('Paul')
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sequel/schema-sharding'
|
3
|
+
|
4
|
+
describe Sequel::SchemaSharding::Ring do
|
5
|
+
|
6
|
+
describe 'a ring of servers' do
|
7
|
+
it 'have the continuum sorted by value' do
|
8
|
+
shards = [1, 2, 3, 4, 5, 6, 7, 8]
|
9
|
+
ring = Sequel::SchemaSharding::Ring.new(shards)
|
10
|
+
previous_value = 0
|
11
|
+
ring.continuum.each do |entry|
|
12
|
+
expect(entry.value).to be > previous_value
|
13
|
+
previous_value = entry.value
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
describe '#shard_for_id' do
|
19
|
+
it 'returns a server for a given id' do
|
20
|
+
shards = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
|
21
|
+
ring = Sequel::SchemaSharding::Ring.new(shards)
|
22
|
+
expect(ring.shard_for_id(3489409)).to eq(4)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
ENV['RACK_ENV'] ||= 'test'
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
Bundler.require 'test'
|
5
|
+
|
6
|
+
require 'sequel-schema-sharding'
|
7
|
+
require 'support/database_helper'
|
8
|
+
require 'mocha/api'
|
9
|
+
|
10
|
+
RSpec.configure do |config|
|
11
|
+
config.treat_symbols_as_metadata_keys_with_true_values = true
|
12
|
+
config.run_all_when_everything_filtered = true
|
13
|
+
config.filter_run :focus
|
14
|
+
config.alias_example_to :fit, focus: true
|
15
|
+
config.mock_framework = :mocha
|
16
|
+
|
17
|
+
|
18
|
+
# Run specs in random order to surface order dependencies. If you find an
|
19
|
+
# order dependency and want to debug it, you can fix the order by providing
|
20
|
+
# the seed, which is printed after each run.
|
21
|
+
# --seed 1234
|
22
|
+
config.order = 'random'
|
23
|
+
|
24
|
+
config.before :all do |ex|
|
25
|
+
Sequel::SchemaSharding.logger = Logger.new(StringIO.new)
|
26
|
+
Sequel::SchemaSharding.sharding_yml_path = "spec/fixtures/test_db_config.yml"
|
27
|
+
Sequel::SchemaSharding.migration_path = "spec/fixtures/db/migrate"
|
28
|
+
end
|
29
|
+
|
30
|
+
config.around :each do |ex|
|
31
|
+
#Sequel::SchemaSharding.config = Sequel::SchemaSharding::Configuration.new('boom', 'spec/fixtures/test_db_config.yml')
|
32
|
+
|
33
|
+
# Start transactions in each connection to the physical shards
|
34
|
+
connections = Sequel::SchemaSharding.config.physical_shard_configs.map do |shard_config|
|
35
|
+
Sequel::SchemaSharding.connection_manager[shard_config[0]]
|
36
|
+
end
|
37
|
+
|
38
|
+
start_transaction_proc = Proc.new do |connections|
|
39
|
+
if connections.length == 0
|
40
|
+
ex.run
|
41
|
+
else
|
42
|
+
connections[0].transaction do
|
43
|
+
connections.shift
|
44
|
+
start_transaction_proc.call(connections)
|
45
|
+
raise Sequel::Rollback
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
start_transaction_proc.call(connections)
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
|
3
|
+
class DatabaseHelper
|
4
|
+
def self.db
|
5
|
+
@db = Sequel.postgres(user: 'postgres', password: 'alkfjasdlfj', host: 'localhost')
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.disconnect
|
9
|
+
db.disconnect
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.drop_db(database)
|
13
|
+
if db_exists?("#{database}")
|
14
|
+
self.disconnect
|
15
|
+
|
16
|
+
db.run("DROP DATABASE #{database}")
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.db_exists?(database)
|
21
|
+
dbs.include?(database)
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.dbs
|
25
|
+
db.fetch('SELECT datname FROM pg_database WHERE datistemplate = false;').all.map { |d| d[:datname] }
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.schema_exists?(database, schema)
|
29
|
+
schemas(database).include?(schema)
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.schemas(database)
|
33
|
+
Sequel::SchemaSharding.connection_manager[database].fetch('select nspname from pg_catalog.pg_namespace;').all.map { |d| d[:nspname] }
|
34
|
+
end
|
35
|
+
end
|
metadata
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sequel-schema-sharding
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Paul Henry
|
8
|
+
- James Hart
|
9
|
+
- Eric Saxby
|
10
|
+
autorequire:
|
11
|
+
bindir: bin
|
12
|
+
cert_chain: []
|
13
|
+
date: 2013-09-06 00:00:00.000000000 Z
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: sequel
|
17
|
+
requirement: !ruby/object:Gem::Requirement
|
18
|
+
requirements:
|
19
|
+
- - '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
requirements:
|
26
|
+
- - '>='
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
version: '0'
|
29
|
+
- !ruby/object:Gem::Dependency
|
30
|
+
name: pg
|
31
|
+
requirement: !ruby/object:Gem::Requirement
|
32
|
+
requirements:
|
33
|
+
- - '>='
|
34
|
+
- !ruby/object:Gem::Version
|
35
|
+
version: '0'
|
36
|
+
type: :runtime
|
37
|
+
prerelease: false
|
38
|
+
version_requirements: !ruby/object:Gem::Requirement
|
39
|
+
requirements:
|
40
|
+
- - '>='
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: '0'
|
43
|
+
- !ruby/object:Gem::Dependency
|
44
|
+
name: bundler
|
45
|
+
requirement: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - ~>
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: '1.3'
|
50
|
+
type: :development
|
51
|
+
prerelease: false
|
52
|
+
version_requirements: !ruby/object:Gem::Requirement
|
53
|
+
requirements:
|
54
|
+
- - ~>
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: '1.3'
|
57
|
+
- !ruby/object:Gem::Dependency
|
58
|
+
name: rake
|
59
|
+
requirement: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - '>='
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: '0'
|
64
|
+
type: :development
|
65
|
+
prerelease: false
|
66
|
+
version_requirements: !ruby/object:Gem::Requirement
|
67
|
+
requirements:
|
68
|
+
- - '>='
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '0'
|
71
|
+
description: ''
|
72
|
+
email:
|
73
|
+
- dev@wanelo.com
|
74
|
+
executables: []
|
75
|
+
extensions: []
|
76
|
+
extra_rdoc_files: []
|
77
|
+
files:
|
78
|
+
- .gitignore
|
79
|
+
- .rspec
|
80
|
+
- .travis.yml
|
81
|
+
- Gemfile
|
82
|
+
- LICENSE.txt
|
83
|
+
- README.md
|
84
|
+
- Rakefile
|
85
|
+
- lib/sequel-schema-sharding.rb
|
86
|
+
- lib/sequel/schema-sharding.rb
|
87
|
+
- lib/sequel/schema-sharding/configuration.rb
|
88
|
+
- lib/sequel/schema-sharding/connection_manager.rb
|
89
|
+
- lib/sequel/schema-sharding/database_manager.rb
|
90
|
+
- lib/sequel/schema-sharding/finder.rb
|
91
|
+
- lib/sequel/schema-sharding/model.rb
|
92
|
+
- lib/sequel/schema-sharding/ring.rb
|
93
|
+
- lib/sequel/schema-sharding/sequel_ext.rb
|
94
|
+
- lib/sequel/schema-sharding/version.rb
|
95
|
+
- lib/sequel/tasks/test.rake
|
96
|
+
- sequel-schema-sharding.gemspec
|
97
|
+
- spec/fixtures/db/migrate/artists/001_create_test_table.rb
|
98
|
+
- spec/fixtures/db/migrate/boof/001_create_boof_table.rb
|
99
|
+
- spec/fixtures/test_db_config.yml
|
100
|
+
- spec/schema-sharding/configuration_spec.rb
|
101
|
+
- spec/schema-sharding/connection_manager_spec.rb
|
102
|
+
- spec/schema-sharding/database_manager_spec.rb
|
103
|
+
- spec/schema-sharding/finder_spec.rb
|
104
|
+
- spec/schema-sharding/model_spec.rb
|
105
|
+
- spec/schema-sharding/ring_spec.rb
|
106
|
+
- spec/spec_helper.rb
|
107
|
+
- spec/support/database_helper.rb
|
108
|
+
homepage: https://github.com/wanelo/sequel-schema-sharding
|
109
|
+
licenses:
|
110
|
+
- MIT
|
111
|
+
metadata: {}
|
112
|
+
post_install_message:
|
113
|
+
rdoc_options: []
|
114
|
+
require_paths:
|
115
|
+
- lib
|
116
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
117
|
+
requirements:
|
118
|
+
- - '>='
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: '0'
|
121
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
122
|
+
requirements:
|
123
|
+
- - '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
requirements: []
|
127
|
+
rubyforge_project:
|
128
|
+
rubygems_version: 2.0.7
|
129
|
+
signing_key:
|
130
|
+
specification_version: 4
|
131
|
+
summary: Create horizontally sharded Sequel models with Postgres
|
132
|
+
test_files:
|
133
|
+
- spec/fixtures/db/migrate/artists/001_create_test_table.rb
|
134
|
+
- spec/fixtures/db/migrate/boof/001_create_boof_table.rb
|
135
|
+
- spec/fixtures/test_db_config.yml
|
136
|
+
- spec/schema-sharding/configuration_spec.rb
|
137
|
+
- spec/schema-sharding/connection_manager_spec.rb
|
138
|
+
- spec/schema-sharding/database_manager_spec.rb
|
139
|
+
- spec/schema-sharding/finder_spec.rb
|
140
|
+
- spec/schema-sharding/model_spec.rb
|
141
|
+
- spec/schema-sharding/ring_spec.rb
|
142
|
+
- spec/spec_helper.rb
|
143
|
+
- spec/support/database_helper.rb
|