sequel-schema-sharding 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9a83484015fdcd8dc4799e752d6d82ee94498738
-  data.tar.gz: 2e44e0d65a6c598371035daddc1724022b3c893b
+  metadata.gz: 59427115172509c05a9c50ff6b52f6bf13b3bae7
+  data.tar.gz: 82f16ec31818221234a0873abe6663395137e0ba
 SHA512:
-  metadata.gz: bac9d7a0ed39dbe558e2f0bd6422bda34f68676662ba5283722340c1ffbf784f6e8c433ca6b118efe41c8a0e66ed123c8bfd2f482597ace1ab750a2b7d2fcbd1
-  data.tar.gz: 724cae47d487ee6e63a8e969ae8ed9f1d380ffaf61b3877074578a9704565fd0290ffd27550904afc9ad3949aba536b4fa69d5209670a39b13f21ae2e0fd874c
+  metadata.gz: 8620ce75eb60600a686dbd2c3ef53bf5d11d604963e6315e7d467c046578a6c7d76fe408cba8a001925c40ef04ec6576e1c697b4aad821dcc5ffbe3b0a5ebec9
+  data.tar.gz: cd8c89a4f8a40afd9c062aeed3423638db1f7441eb8c9bb967a7a470a134d2fa583d8a53782f6bb526d2cbdfe03570926f5cf36447b85106938d134ee3970f21

data/CONTRIBUTORS.md

@@ -0,0 +1,6 @@
+Contributors
+============
+
+* James Hart (james@wanelo.com)
+* Paul Henry (paul@wanelo.com)
+* Eric Saxby (sax@wanelo.com)

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 TODO: Write your name
+Copyright (c) 2013 Wanelo, Inc
 
 MIT License
 

data/README.md

@@ -1,28 +1,265 @@
-Sequel::SchemaSharding
-================
+sequel-schema-sharding
+======================
 
-[![Build Status](https://travis-ci.org/wanelo/sequel-sharding.png?branch=master)](https://travis-ci.org/wanelo/sequel-sharding)
+[![Build Status](https://travis-ci.org/wanelo/sequel-schema-sharding.png?branch=master)](https://travis-ci.org/wanelo/sequel-schema-sharding)
+
+Horizontally shard PostgreSQL tables with the Sequel gem, where each shard
+lives in its own PostgreSQL schema.
+
+This gem allows you to configure mappings between logical and physical shards, pooling
+connections between logical shards on the same physical server.
 
-Horizontally shard postgres with the Sequel gem. This gem allows you to configure mappings between logical and
-physical shards, and pool connections between logical shards on the same physical server.
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
-    gem 'sequel-sharding'
+    gem 'sequel-schema-sharding'
 
 And then execute:
 
     $ bundle
 
-Or install it yourself as:
-
-    $ gem install sequel-sharding
 
 ## Usage
 
-TODO :)
+See the `examples` directory for example files.
+
+### Configuration
+
+Create a sharding configuration file in your project, for instance at
+`config/sharding.yml`. The format should match the following
+conventions:
+
+```yml
+<env>:
+  tables:
+    <table_name>:
+      schema_name: "schema_%e_%s"
+      logical_shards:
+        <1..n>: <shard_name>
+  physical_shards:
+    <shard_name>:
+      host: <hostname>
+      database: <database>
+  common:
+    username: <pg_username>
+    password: <pg_password>
+    port: <pg_port>
+```
+
+Tables can coexist in schemas, though they do not have to.
+
+In your project, configure `sequel-schema-sharding` in a ruby file that
+gets loaded before your models, for instance at `config/sharding.rb`.
+
+```ruby
+require 'sequel-schema-sharding'
+
+Sequel::SchemaSharding.migration_path = File.expand_path('../../db/sharding_migrations', __FILE__)
+Sequel::SchemaSharding.sharding_yml_path = File.expand_path('../sharding.yml', __FILE__)
+```
+
+### Migrations
+
+Each table gets its own set of migrations. Underneath the scenes,
+`sequel-schema-sharding` uses Sequel migrations, though migrations are
+run using the `Sequel::SchemaSharding::DatabaseManager` class.
+
+For instance, if you have two sharded tables, `:artists` and `:albums`,
+your migration folder would look something like this:
+
+```yml
+- my_project
+  - db
+    - migrations
+      - artists
+        - 001_create_artists.rb
+        - 002_add_indexes_to_artists.rb
+      - albums
+        - 001_create_albums.rb
+```
+
+See Sequel documentation for more info:
+* (http://sequel.rubyforge.org/rdoc/files/doc/schema_modification_rdoc.html)
+* (http://sequel.rubyforge.org/rdoc/files/doc/migration_rdoc.html)
+
+TODO: rake tasks for running migrations
+
+### Models
+
+Models declare their table in the class definition. This allows Sequel
+to load table information from the database when the environment loads.
+This is particularly important for typecasting, so empty strings can be
+typecast to null, etc.
+
+The tricky bit is that `sequel-schema-sharding` connects to the first
+available shard for a table in order to read the database schema.
+
+```ruby
+require 'config/sharding'
+
+class Artist < Sequel::SchemaSharding::Model('artists')
+  set_columns [:id, :name]
+  set_sharded_column :id
+
+  def this
+    @this ||= self.class.by_id(id)
+  end
+
+  def self.by_id(id)
+    shard_for(id).where(id: id).first
+  end
+end
+
+class Album < Sequel::SchemaSharding::Model('albums')
+  set_columns [:artist_id, :name, :release_date, :created_at]
+  set_sharded_column :artist_id
+
+  def this
+    @this ||= self.class.by_artist(artist_id)
+  end
+
+  def by_artist(artist_id)
+    shard_for(artist_id).where(artist_id: artist_id)
+  end
+
+  def by_artist_and_name(artist_id, name)
+    shard_for(artist_id).where(name: name, artist_id: artist_id)
+  end
+end
+```
+
+Note that logical and physical shards mapped in schema.yml need to exist
+before you can load models into memory.
+
+Read access always starts with the `:shard_for` method, to ensure that
+the correct database connection and shard name is used. Writes will
+automatically choose the correct shard based on the sharded column.
+Never try to insert records with nil values in sharded columns.
+
+TODO: explain why we define `this`
+
+## FAQ
+
+### How should I shard my databases?
+
+This is entirely dependent on the access patterns of your application. A
+good rule, though, is to look at your indexes. If every query goes
+through an index on `:user_id`, then chances are that you should shard
+on `:user_id`. If half of your queries go through `:user_id` and the
+other half go through `:job_id`, then you may need to create two sets of
+shards, each with its own model, and have your application write to
+both. This requires additional application complexity to keep the two
+sets of shards in sync—it's less complex than doing multi-shard reads to
+keep everything in one model, though.
+
+When going into database sharding, an early exercise that is very
+helpful is to analyze application queries and try to reduce the number
+of unique queries. If possible, try to refactor queries such that they
+fit into the smallest number of shard types. For instance, if you find
+Albums by release year, but every action you query from already has the
+`:artist_id`, consider changing your query to find by `:artist_id` and 
+release year.
+
+### How should I generate IDs?
+
+This is also dependent on your application and your comfort level with
+various technologies, but regardless should be done outside
+of `sequel-schema-sharding`. In general there are three approaches that
+we've considered:
+
+* Follow Instagram's approach and let PostgreSQL generate ids. They
+  install functions into each shard, to ensure that each shard generates
+  unique ids.
+
+* Follow Twitter's approach and deploy a separate service for unique id
+  generation. Their in-house solution is called Snowflake, and depends
+  on maven, finagle and thrift.
+
+* Why use ids at all? If you are sharding Like data or something that
+  looks similar to a join table, you may not need a unique identifier.
+  You are probably sharding on a foreign key to some other table, and
+  may not ever access individual Likes by id.
+
+### Should each table get its own set of shards/schemas?
+
+In the early days of a project's lifetime, it may seem like less
+management overhead to let multiple tables coexist in each shard.
+Experience with sharding in other technologies (particularly Redis) have
+shown us that in any sharded data store, you will eventually need
+to redistribute shards. More data equals larger storage and RAM
+requirements, and as servers fill up you will find yourself needing to
+move shards onto a greater number of servers. If your project is
+successful, this may come much sooner than you expect in initial
+infrastructure planning meetings.
+
+Colocating multiple data sets in individual shards makes shard
+redistribution more complicated and risk-prone. More things break when
+an individual shard goes down. Pages or queries that depend on an
+individual data set will stop working when you take down shards to do
+maintenance on other data sets.
+
+Simply put, it's less stressful when doing operational maintenance to
+require twice as many steps that are each easier and less risk-prone.
+So, do whatever you feel is best, but we've chosen to make each shard
+single-purpose in our infrastructure.
+
+### Sequel does sharding. Why another gem?
+
+The sharding plugin that ships with Sequel assumes that each shard is a
+separate database. This means that each shard requires a separate
+connection pool, and that each shard includes every table. When
+splitting a database into thousands of shards, this means that each
+application process requires thousands of connections. A proxy such as
+PGBouncer could help reduce the number of connections from an individual
+application server, but even then PGBouncer would need to manage thousands 
+of connections.
+
+When designing a sharded architecture similar to Instagram's approach 
+(http://instagram-engineering.tumblr.com/post/10853187575/sharding-ids-at-instagram),
+it may be desirable to start with thousands or tens of thousands of shards,
+to delay the need for resharding as long as possible. PostgreSQL is able
+to manage tens of thousands of schemas in a single database without
+significant performance problems, so we can design a sharded backend of
+thousands of shards living on a few physical servers. As stored data
+grows, these shards can be moved onto a greater number of servers,
+without the complication of resharding (i.e. changing the number of
+shards while retaining the exact mapping of data into old shards).
+
+### Why Sequel?
+
+After both good and bad experiences with other Ruby ORMs, Sequel's
+documentation, ease of use and understandable codebase made it a solid
+choice for us. The fact that it already supports horizontal sharding and
+was easy to adapt to our own requirements were a pleasant surprise.
+
+### What the what?? def self.Model; ???
+
+Yeah, this threw us for a while, too. The thing is, ORMs in Ruby tend to
+load information like column info, indexes, etc directly from the
+connected databases, rather than from local schema dictionaries. In
+order to do this, databases need to be created and migrations run BEFORE
+model files can validly loaded.
+
+If the ORM doesn't load this info from somewhere, then it can't
+correctly do things like typecast string HTTP params to integers (or
+nulls).
+
+Rather than monkeypatching our way around this requirement in Sequel, we
+ride the wave and just patch in our additions.
+
+### What could go wrong?
+
+The thing that you never want to happen is to change the mapping of
+shards to data. For instance, if you change the number of shards without
+migrating data into a new database backend, the algorithm by which
+schemas are chosen will start returning a different mapping for reads than
+that which was used to insert data. New records will go into the new
+mapping, but any attempt to read a record inserted via the old mapping
+will pick the wrong shard and return an empty set. DON'T EVER DO THIS.
+It's really embarrassing.
+
 
 ## Contributing
 

data/examples/db/migrations/albums/001_create_albums.rb

@@ -0,0 +1,15 @@
+Sequel.migration do
+  up do
+    create_table :albums do
+      Integer :artist_id, null: false
+      String :name, null: false
+      Date :release_date
+      Time :created_at
+    end
+  end
+
+  down do
+    drop_table :albums
+  end
+end
+

data/examples/db/migrations/artists/001_create_artists.rb

@@ -0,0 +1,15 @@
+Sequel.migration do
+  up do
+    create_table :artists do
+      Integer :id, null: false
+      String :name, null: false
+      Time :created_at
+    end
+  end
+
+  down do
+    drop_table :artists
+  end
+end
+
+

data/examples/model.rb

@@ -0,0 +1,19 @@
+require 'config/sharding'
+
+class Thing < Sequel::SchemaSharding::Model('things')
+  set_columns [:name, :thing1, :thing2]
+  set_sharded_column :name
+
+  # class variables used by Sequel can't easily be set via
+  # pretty methods at the moment. They can be quickly overridden,
+  # however.
+  @require_modification = false
+
+  def this
+    @this ||= self.class.by_name(name)
+  end
+
+  def self.by_name(name)
+    shard_for(name).where(name: name)
+  end
+end

data/examples/sharding.rb

@@ -0,0 +1,4 @@
+require 'sequel-schema-sharding'
+
+Sequel::SchemaSharding.migration_path = File.expand_path('../db/sharding_migrations', __FILE__)
+Sequel::SchemaSharding.sharding_yml_path = File.expand_path('../sharding.yml', __FILE__)

data/examples/sharding.yml

@@ -0,0 +1,53 @@
+test:
+  tables:
+    artists:
+      schema_name: artists_%e_%s
+      logical_shards:
+        1..2: shard1
+        3..4: shard2
+    albums:
+      schema_name: albums_%e_%s
+      logical_shards:
+        1..2: shard2
+        3..4: shard3
+  physical_shards:
+    shard1:
+      host: 127.0.0.1
+      database: my_project_test_shard1
+    shard2:
+      host: 127.0.0.1
+      database: my_project_test_shard2
+    shard3:
+      host: 127.0.0.1
+      database: my_project_test_shard3
+  common:
+    username: postgres
+    password: 
+    port: 5432
+development:
+  tables:
+    artists:
+      schema_name: artists_%e_%s
+      logical_shards:
+        1..2: shard1
+        3..4: shard2
+    albums:
+      schema_name: albums_%e_%s
+      logical_shards:
+        1..2: shard2
+        3..4: shard3
+  physical_shards:
+    shard1:
+      host: 127.0.0.1
+      database: my_project_development_shard1
+    shard2:
+      host: 127.0.0.1
+      database: my_project_development_shard2
+    shard3:
+      host: 127.0.0.1
+      database: my_project_development_shard3
+  common:
+    username: postgres
+    password: 
+    port: 5432
+

data/lib/sequel/schema-sharding/version.rb

@@ -1,5 +1,5 @@
 module Sequel
   module SchemaSharding
-    VERSION = "0.0.1"
+    VERSION = "0.0.2"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sequel-schema-sharding
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Paul Henry
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-06 00:00:00.000000000 Z
+date: 2013-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -78,10 +78,16 @@ files:
 - .gitignore
 - .rspec
 - .travis.yml
+- CONTRIBUTORS.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/db/migrations/albums/001_create_albums.rb
+- examples/db/migrations/artists/001_create_artists.rb
+- examples/model.rb
+- examples/sharding.rb
+- examples/sharding.yml
 - lib/sequel-schema-sharding.rb
 - lib/sequel/schema-sharding.rb
 - lib/sequel/schema-sharding/configuration.rb
@@ -125,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.7
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: Create horizontally sharded Sequel models with Postgres
@@ -141,3 +147,4 @@ test_files:
 - spec/schema-sharding/ring_spec.rb
 - spec/spec_helper.rb
 - spec/support/database_helper.rb
+has_rdoc: