sequel-schema-sharding 0.11.1 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 856de612ab94179af8d327fab5aa6b6f0cd77e5c
-  data.tar.gz: 76cff6fb49277e3f8526980e68cbb165943c1349
+  metadata.gz: e9d2a8dea7d18fc038c01e427503dc01aa6b24f0
+  data.tar.gz: 8033c5b54ac76371fe1f0f3074f75d10d33e14fe
 SHA512:
-  metadata.gz: 9a1e1bcc2b1b010375be855c567ea52f5f776e76f90f64c6fca2f6c3af6a511d639fdeb26016834ec49b94dbb659af30deedec103858f4f1f35a83597e327163
-  data.tar.gz: a0f59b3a1d3344f752997aa15017a5f65fe99d92fa0e89282872259777a73e79019eb801dd4570d12e3558bfd167c3d44bb7b18722224c90459e81cd72289c7b
+  metadata.gz: 56f54ffdeb952181d59b683e74312c3387bc0edeb037c11b2d658d5a653b0e5eef867d44139c56c2ce6f6179d4d0275c1c70efe7abfd7696cfcbe837605547cb
+  data.tar.gz: 3d980233dd79fd9eac01f56550885d3a02589f1b0b95718dbad2d47aa5d5750b7225f134819a871759e3d9a3f4e4805f4e496ca82b4ee35688f63e9fc76c4ab4

data/Gemfile

@@ -8,7 +8,4 @@ group :test do
   gem 'mocha', require: false
   gem 'pry-nav'
   gem 'guard-rspec'
-
-  # for testing purposes until pull request is accepted
-  gem 'ruby-usdt', github: 'sax/ruby-usdt', ref: 'probe-introspection', submodules: true
 end

data/README.md

@@ -1,7 +1,9 @@
 sequel-schema-sharding
 ======================
 
+[![Gem Version](https://badge.fury.io/rb/sequel-schema-sharding.png)](http://badge.fury.io/rb/sequel-schema-sharding)
 [![Build Status](https://travis-ci.org/wanelo/sequel-schema-sharding.png?branch=master)](https://travis-ci.org/wanelo/sequel-schema-sharding)
+[![Code Climate](https://codeclimate.com/github/wanelo/sequel-schema-sharding.png)](https://codeclimate.com/github/wanelo/sequel-schema-sharding)
 
 Horizontally shard PostgreSQL tables with the Sequel gem, where each shard
 lives in its own PostgreSQL schema.
@@ -201,7 +203,7 @@ queries do not try to reconnect to a downed master.
 
 ```bash
 > bundle install
-> bundle exec rake sequel:db:create
+> bundle exec rake sequel:db:test:reset
 > bundle exec rspec
 ```
 
@@ -325,6 +327,18 @@ mapping, but any attempt to read a record inserted via the old mapping
 will pick the wrong shard and return an empty set. DON'T EVER DO THIS.
 It's really embarrassing.
 
+### Any problems with other services?
+
+When integrating with NewRelic, *do not* enable the SQL query plan
+instrumentation. It can grab a connection that your application is also
+trying to use... libpq is thread safe, so long as two threads do not
+try to manipulate the same PGonn object
+(http://www.postgresql.org/docs/9.3/static/libpq-threading.html).
+If you see errors such as `PG::UnableToSend: insufficient data in "T" message`
+or `PG::UnableToSend: extraneous data in "T" message`, this can indicate that
+multiple threads are accessing the same connection, and data (or random bytes)
+may have been transposed between queries.
+
 
 ## Contributing
 

data/lib/sequel/plugins/dataset_shard_id.rb

@@ -0,0 +1,21 @@
+module Sequel
+  module Plugins
+    module DatasetShardId
+      module InstanceMethods
+        def shard_number
+          @values[:shard_number] if @values
+        end
+      end
+
+      module DatasetMethods
+        def shard_number
+          @shard_number
+        end
+
+        def shard_number=(id)
+          @shard_number=id
+        end
+      end
+    end
+  end
+end

data/lib/sequel/schema-sharding/connection_manager.rb

@@ -77,16 +77,13 @@ module Sequel
       def replica_hash_for(config)
         return {} if config['replicas'].nil?
         size = config['replicas'].size
-        i = rand(size)
         {
           :servers => {
             :read_only => ->(db) do
-              choice = i % size
+              choice = rand(size)
               probe = Sequel::SchemaSharding::DTraceProvider.provider.replica_hash_for
               probe.fire(choice, size) if probe.enabled?
-              sequel_connection_config_for(config['replicas'][choice]).tap do
-                i += 1
-              end
+              sequel_connection_config_for(config['replicas'][choice])
             end
           }
         }

data/lib/sequel/schema-sharding/extensions/postgres/dataset.rb

@@ -0,0 +1,16 @@
+module Sequel
+  module Postgres
+    class Dataset
+      alias_method :adapter_fetch_rows, :fetch_rows
+
+      def fetch_rows(sql, &block)
+        adapter_fetch_rows(sql) do |r|
+          if self.respond_to?(:shard_number)
+            r[:shard_number] = self.shard_number
+          end
+          block.call r
+        end
+      end
+    end
+  end
+end

data/lib/sequel/schema-sharding/extensions/sequel_ext.rb

@@ -1,16 +1,20 @@
 module Sequel
-  module SchemaSharding
-    module Extensions
-      module SequelExt
-        module ClassMethods
-          def db
-            return @db if @db
-          end
+  class Database
+    class << self
+      alias_method :sequel_adapter_class, :adapter_class
+
+      def adapter_class(scheme)
+        klass = sequel_adapter_class(scheme)
+
+        begin
+          require "sequel/schema-sharding/extensions/#{scheme}/dataset"
+        rescue LoadError => e
         end
+
+        klass
       end
     end
   end
 end
 
-Sequel::Model.plugin Sequel::SchemaSharding::Extensions::SequelExt
 Sequel::Model.plugin :validation_helpers

data/lib/sequel/schema-sharding/finder.rb

@@ -4,11 +4,12 @@ module Sequel
   module SchemaSharding
     class Finder
       class Result
-        attr_reader :connection, :schema
+        attr_reader :connection, :schema, :shard_number
 
-        def initialize(connection, schema)
+        def initialize(connection, schema, shard_number)
           @connection = connection
           @schema = schema
+          @shard_number = shard_number
         end
       end
 
@@ -21,7 +22,7 @@ module Sequel
         conn = Sequel::SchemaSharding.connection_manager[physical_shard]
         schema = Sequel::SchemaSharding.connection_manager.schema_for(table_name, shard_number)
 
-        Result.new(conn, schema)
+        Result.new(conn, schema, shard_number)
       end
 
       private

data/lib/sequel/schema-sharding/model.rb

@@ -28,6 +28,7 @@ module Sequel
 
       def self.included(base)
         base.extend(ClassMethods)
+        base.plugin(:dataset_shard_id)
       end
 
       module ClassMethods
@@ -53,6 +54,7 @@ module Sequel
           ds = result.connection[schema_and_table(result)]
           ds.row_proc = self
           dataset_method_modules.each { |m| ds.instance_eval { extend(m) } }
+          ds.shard_number = result.shard_number
           ds.model = self
           ds
         end
@@ -71,6 +73,14 @@ module Sequel
         def schema_and_table(result)
           :"#{result.schema}__#{self.implicit_table_name}"
         end
+
+        def create(values = {}, &block)
+          sharded_column_value = values[sharded_column]
+          shard_number = result_for(sharded_column_value).shard_number
+          super.tap do |m|
+            m.values[:shard_number] = shard_number
+          end
+        end
       end
 
       # The database connection that has the logical shard.

data/lib/sequel/schema-sharding/version.rb

@@ -1,5 +1,5 @@
 module Sequel
   module SchemaSharding
-    VERSION = "0.11.1"
+    VERSION = "0.12.0"
   end
 end

data/lib/sequel/tasks/test.rake

@@ -20,5 +20,14 @@ namespace :sequel do
       manager = Sequel::SchemaSharding::DatabaseManager.new
       manager.drop_databases
     end
+
+    namespace :test do
+      desc 'Reset test database'
+      task :reset do
+        ENV['RACK_ENV'] = 'test'
+        Rake::Task['sequel:db:drop'].invoke
+        Rake::Task['sequel:db:create'].invoke
+      end
+    end
   end
 end

data/sequel-schema-sharding.gemspec

@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'sequel'
   spec.add_dependency 'pg'
   spec.add_dependency 'sequel-replica-failover'
-  spec.add_dependency 'ruby-usdt'
+  spec.add_dependency 'ruby-usdt', '>= 0.2.1'
 
   spec.add_development_dependency 'bundler', '~> 1.3'
   spec.add_development_dependency 'rake'

data/spec/schema-sharding/finder_spec.rb

@@ -9,6 +9,7 @@ describe Sequel::SchemaSharding::Finder do
       result = Sequel::SchemaSharding::Finder.instance.lookup('boof', 60)
       expect(result.connection).to be_a(Sequel::Postgres::Database)
       expect(result.schema).to eq('sequel_logical_boof_01')
+      expect(result.shard_number).to eq(1)
     end
 
     xit 'is fast' do

data/spec/schema-sharding/model_spec.rb

@@ -25,8 +25,8 @@ describe Sequel::SchemaSharding, 'Model' do
     klass
   end
 
-  describe '#by_id' do
-    it 'returns a valid artist by id' do
+  describe 'reading from database' do
+    it 'can return a valid record' do
       artist = model.create(artist_id: 14, name: 'Paul')
       expect(artist.id).to_not be_nil
       read_back_artist = model.by_id(14).first
@@ -36,15 +36,28 @@ describe Sequel::SchemaSharding, 'Model' do
       read_back_artist = model.by_id(14).first
       expect(read_back_artist).to be_nil
     end
+
+    it 'includes shard number on model instances' do
+      shard_number = model.shard_for(456).shard_number
+
+      model.create(artist_id: 456, name: 'Randy')
+      record = model.by_id(456).first
+      expect(record.shard_number).to eq(shard_number)
+    end
   end
 
-  describe '#create' do
-    it 'creates a valid artist' do
+  describe 'writing to database' do
+    it 'can create a valid record' do
       artist = model.create(artist_id: 234, name: 'Paul')
       expect(artist).to be_a(model)
       expect(artist.name).to eql('Paul')
       artist.destroy
     end
+
+    it 'includes shard number on model instances' do
+      shard_number = model.shard_for(5432).shard_number
+      expect(model.create(artist_id: 5432, name: 'WOW').shard_number).to eq(shard_number)
+    end
   end
 
   describe '#shard_for' do
@@ -58,6 +71,10 @@ describe Sequel::SchemaSharding, 'Model' do
       expect(dataset.db.opts[:database]).to eq('sequel_test_shard2')
       expect(dataset.first_source).to eq(:sequel_logical_artists_17__artists)
     end
+
+    it 'includes shard_number on dataset' do
+      expect(dataset.shard_number).to eq(17)
+    end
   end
 
   describe '#read_only_shard_for' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sequel-schema-sharding
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.12.0
 platform: ruby
 authors:
 - Paul Henry
@@ -10,90 +10,90 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-05 00:00:00.000000000 Z
+date: 2014-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: sequel-replica-failover
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-usdt
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: ''
@@ -103,9 +103,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .travis.yml
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
 - CONTRIBUTORS.md
 - Gemfile
 - Guardfile
@@ -118,6 +118,7 @@ files:
 - examples/sharding.rb
 - examples/sharding.yml
 - lib/sequel-schema-sharding.rb
+- lib/sequel/plugins/dataset_shard_id.rb
 - lib/sequel/schema-sharding.rb
 - lib/sequel/schema-sharding/configuration.rb
 - lib/sequel/schema-sharding/connection_manager.rb
@@ -125,6 +126,7 @@ files:
 - lib/sequel/schema-sharding/database_manager/schema_iterator.rb
 - lib/sequel/schema-sharding/dtrace_provider.rb
 - lib/sequel/schema-sharding/extensions/migrations_ext.rb
+- lib/sequel/schema-sharding/extensions/postgres/dataset.rb
 - lib/sequel/schema-sharding/extensions/sequel_ext.rb
 - lib/sequel/schema-sharding/finder.rb
 - lib/sequel/schema-sharding/logger_proxy.rb
@@ -158,17 +160,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.7
+rubygems_version: 2.2.0
 signing_key: 
 specification_version: 4
 summary: Create horizontally sharded Sequel models with Postgres