em-pg-sequel 0.0.3 → 0.0.4

data/Gemfile

@@ -2,8 +2,8 @@ source 'https://rubygems.org'
 
 group :test do
   gem "sequel"
-  gem "minitest"
-  gem "minitest-reporters"
+  gem "minitest", "~> 4"
+  gem "minitest-reporters", "< 1"
   gem "debugger"
 end
 

data/README.md

@@ -8,9 +8,10 @@ Usage
 
 ```ruby
 require "em-pg-sequel"
+require "em-synchrony"
 EM.synchrony do
   url = "postgres://postgres:postgres@localhost:5432/test"
-  db = Sequel.connect(url, pool_class: EM::PG::ConnectionPool)
+  db = Sequel.connect(url, pool_class: :em_synchrony)
 
   puts db[:test].all.inspect
 

data/em-pg-sequel.gemspec

@@ -4,8 +4,8 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "em-pg-sequel"
-  gem.version       = "0.0.3"
-  gem.authors       = ["Petr Yanovich"]
+  gem.version       = "0.0.4"
+  gem.authors       = ["Petr Yanovich", "Rafal Michalski"]
   gem.email         = ["fl00r@yandex.ru"]
   gem.description   = %q{Sequel adapter for ruby-em-pg-client}
   gem.summary       = %q{Sequel adapter for ruby-em-pg-client}
@@ -17,8 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_dependency "sequel"
-  gem.add_dependency "em-synchrony"
-  gem.add_dependency "em-pg-client"
+  gem.add_dependency "em-pg-client", ">= 0.3.0"
 
   gem.add_development_dependency "em-synchrony"
 end

data/lib/em-pg-sequel.rb

@@ -1,53 +1,10 @@
-require 'em-synchrony'
-require 'em-synchrony/pg'
+require 'em-pg-client'
 require 'sequel'
 require 'em-pg-sequel/connection_pool'
 
-module EM::PG
-  class ConnectionPool < ::Sequel::ConnectionPool
-    DEFAULT_SIZE = 4
-    attr_accessor :pool
-    def initialize(db, opts = {})
-      super
-      size = opts[:max_connections] || DEFAULT_SIZE
-      @pool = ::EM::PG::Sequel::ConnectionPool.new(size: size, disconnect_class: ::Sequel::DatabaseConnectionError) do
-        make_new(DEFAULT_SERVER)
-      end
-    end
-
-    def size
-      @pool.available.size
-    end
-
-    def hold(server = nil, &blk)
-      @pool.execute(&blk)
-    end
-
-    def disconnect(server = nil)
-      @pool.available.each{ |conn| db.disconnect_connection(conn) }
-      @pool.available.clear
-    end
-  end
-end
-
-module PG::EM
-  class SyncClient < Client
-    # Dirty hack
-    # To avoid patching ruby-em-pg-client and to support Sequel API
-    # we should execute async_client asynchronously for em-pg and synchronously for sequel
-    def async_exec(*args)
-      if block_given?
-        super
-      else
-        exec(*args)
-      end
-    end
-  end
-end
-
 $VERBOSE.tap do |old_verbose|
   $VERBOSE = nil
-  PGconn = PG::EM::SyncClient
+  PGconn = PG::EM::Client
   $VERBOSE = old_verbose
 end
 

data/lib/em-pg-sequel/connection_pool.rb

@@ -1,47 +1,97 @@
 module EM::PG
   module Sequel
-    class ConnectionPool
-      attr_reader :available
+    class ConnectionPool < ::Sequel::ConnectionPool
 
-      def initialize(opts, &blk)
+      DEFAULT_SIZE = 4
+
+      attr_reader :available, :allocated, :max_size
+
+      def initialize(db, opts = {})
+        super
         @available = []
+        @allocated = {}
         @pending = []
-        @acquire_blk = blk
 
-        @disconnected_class = opts[:disconnect_class]
+        @max_size = opts[:max_connections] || DEFAULT_SIZE
+        hold {}
+      end
+
+      def size
+        @available.length + @allocated.length
+      end
+
+      def hold(server = nil)
+        fiber = Fiber.current
+        fiber_id = fiber.object_id
+
+        if conn = @allocated[fiber_id]
+          skip_release = true
+        else
+          conn = acquire(fiber) until conn
+        end
+
+        begin
+          yield conn
+
+        rescue ::Sequel::DatabaseDisconnectError => e
+          db.disconnect_connection(conn)
+          drop_failed(fiber_id)
+          skip_release = true
 
-        opts[:size].times do
-          @available.push @acquire_blk.call
+          raise
+        ensure
+          release(fiber_id) unless skip_release
         end
       end
 
-      def execute
-        conn = acquire
-        yield conn
-      rescue => e
-        conn = @acquire_blk.call if @disconnected_class && @disconnected_class === e
-        raise
-      ensure
-        release(conn)
+      def disconnect(server = nil)
+        @available.each{ |conn| db.disconnect_connection(conn) }
+        @available.clear
       end
 
-      def acquire
-        f = Fiber.current
+      private
+
+      def acquire(fiber)
         if conn = @available.pop
-          conn
+          @allocated[fiber.object_id] = conn
         else
-          @pending << f
-          Fiber.yield
+          if size < max_size
+            allocate_new_connection(fiber.object_id)
+          else
+            @pending << fiber
+            Fiber.yield
+          end
         end
       end
 
-      def release(conn)
-        if job = @pending.shift
-          EM.next_tick{ job.resume conn }
+      def allocate_new_connection(fiber_id)
+        @allocated[fiber_id] = true
+        @allocated[fiber_id] = make_new(DEFAULT_SERVER)
+      rescue Exception => e
+        drop_failed(fiber_id)
+        raise e
+      end
+
+      # drop failed connection (or a mark) from the pool and
+      # ensure that the pending requests won't starve
+      def drop_failed(fiber_id)
+        @allocated.delete(fiber_id)
+        if pending = @pending.shift
+          EM.next_tick { pending.resume }
+        end
+      end
+
+      def release(fiber_id)
+        conn = @allocated.delete(fiber_id)
+        if pending = @pending.shift
+          @allocated[pending.object_id] = conn
+          EM.next_tick { pending.resume conn}
         else
           @available << conn
         end
       end
+
+      CONNECTION_POOL_MAP[:em_synchrony] = self
     end
   end
 end

data/spec/em-pg-sequel/sequel_spec.rb

@@ -3,20 +3,44 @@ require 'em-synchrony'
 require 'em-synchrony/fiber_iterator'
 
 describe EM::PG::Sequel do
+  include SynchronyUtils
+
   DELAY = 1
   QUERY = "select pg_sleep(#{DELAY})"
 
   let(:url) { DB_URL }
   let(:size) { 1 }
-  let(:db) { Sequel.connect(url, max_connection: size, pool_class: EM::PG::ConnectionPool, db_logger: Logger.new(nil)) }
+  let(:db) { Sequel.connect(url, max_connections: size, pool_class: :em_synchrony, db_logger: Logger.new(nil)) }
   let(:test) { db[:test] }
+  let(:fiber_iterator) { EM::Synchrony::FiberIterator }
+
+  describe "sanity" do
+    let(:size) { 42 }
+
+    it "should have max_size 42" do
+      db.pool.max_size.must_equal 42
+    end
+
+    it "should not release nil connection on connect error" do
+      synchrony do
+        db.disconnect
+        db.pool.size.must_equal 0
+        db.pool.stub :allocate_new_connection,
+            proc { raise Sequel::DatabaseConnectionError } do
+
+          proc { test.count }.must_raise Sequel::DatabaseConnectionError
+          db.pool.size.must_equal 0
+        end
+
+      end
+    end
+  end
 
   describe "unexist table" do
     it "should raise exception" do
-      EM.synchrony do
+      synchrony do
         proc { test.all }.must_raise Sequel::DatabaseError
 
-        EM.stop
       end
     end
   end
@@ -24,30 +48,27 @@ describe EM::PG::Sequel do
   describe "exist table" do
 
     before do
-      EM.synchrony do
+      synchrony do
         db.create_table!(:test) do
           text :name
           integer :value, index: true
         end
 
-        EM.stop
       end
     end
 
     after do
-      EM.synchrony do
+      synchrony do
         db.drop_table?(:test)
 
-        EM.stop
       end
     end
 
     it "should connect and execute query" do
-      EM.synchrony do 
+      synchrony do
         test.insert name: "andrew", value: 42
         test.where(name: "andrew").first[:value].must_equal 42
 
-        EM.stop
       end
     end
 
@@ -55,17 +76,16 @@ describe EM::PG::Sequel do
     describe "pool size is exceeded" do
       let(:size) { 1 }
       it "should queue requests" do
-        EM.synchrony do
+        synchrony do
           start = Time.now.to_f
 
           res = []
-          EM::Synchrony::FiberIterator.new([1,2], 1).each do |t|
+          fiber_iterator.new([1,2], 1).each do |t|
             res << db[QUERY].all
           end
           (Time.now.to_f - start.to_f).must_be_within_delta DELAY * 2, DELAY * 2 * 0.15
           res.size.must_equal 2
 
-          EM.stop
         end
       end
     end
@@ -73,18 +93,171 @@ describe EM::PG::Sequel do
     describe "pool size is enough" do
       let(:size) { 2 }
       it "should parallel requests" do
-        EM.synchrony do
+        synchrony do
           start = Time.now.to_f
 
           res = []
-          EM::Synchrony::FiberIterator.new([1,2], 2).each do |t|
+          fiber_iterator.new([1,2], 2).each do |t|
             res << db[QUERY].all
           end
 
           (Time.now.to_f - start.to_f).must_be_within_delta DELAY, DELAY * 0.30
           res.size.must_equal 2
 
-          EM.stop
+        end
+      end
+    end
+
+    describe "pool size is dynamic" do
+
+      let(:size) { 2 }
+
+      it "should have initial size of one" do
+        db.pool.size.must_equal 1
+      end
+
+      it "should allocate second connection" do
+        synchrony do
+          res = []
+          res << test.first
+          db.pool.size.must_equal 1
+          fiber_iterator.new([1,2], 2).each do |t|
+            res << db[QUERY].all
+          end
+          db.pool.size.must_equal 2
+          res.size.must_equal 3
+
+        end
+      end
+
+      it "should not create more than size connections" do
+        synchrony do
+          db.pool.size.must_equal 1
+
+          start = Time.now.to_f
+          res = []
+          fiber_iterator.new([1,1,2], 3).each do |pool_size|
+            db.pool.size.must_equal pool_size
+            res << db[QUERY].all
+          end
+
+          (Time.now.to_f - start.to_f).must_be_within_delta DELAY*2, DELAY * 2.60
+          res.size.must_equal 3
+
+          db.pool.size.must_equal size
+
+        end
+      end
+
+      it "should clear all connections on disconnect" do
+        synchrony do
+          db.disconnect
+          db.pool.size.must_equal 0
+          res = []
+          fiber_iterator.new([1,2,3], 3).each do |t|
+            res << test.count
+          end
+          res.size.must_equal 3
+          db.pool.size.must_equal size
+          db.disconnect
+          db.pool.size.must_equal 0
+
+        end
+      end
+
+      it "should re-create 1st connection" do
+        synchrony do
+          db.disconnect
+          db.pool.size.must_equal 0
+
+          test.count.must_equal 0
+          db.pool.size.must_equal 1
+
+        end
+      end
+
+    end
+
+    describe "on connection errors" do
+
+      let(:size) { 3 }
+
+      it "should not leave pending requests in queue" do
+        synchrony do
+          db.disconnect
+          fiber_iterator.new((0..size), size).each { test.count }
+          db.pool.available.each do |conn|
+            # force clients to disconnected state
+            conn.async_command_aborted = true
+          end.length.must_equal db.pool.size
+
+          db.pool.stub :make_new,
+              proc {
+                EM::Synchrony.sleep 0.1
+                raise Sequel::DatabaseConnectionError } do
+
+            request_counter = 0
+            expected_runs = db.pool.max_size + 10
+            expected_runs.times do |index|
+              Fiber.new do
+
+                pending = db.pool.instance_eval { @pending.length }
+                pending.must_equal [index - db.pool.max_size, 0].max
+
+                if index < db.pool.max_size
+                  proc { test.count }.must_raise Sequel::DatabaseDisconnectError
+                else
+                  proc { test.count }.must_raise Sequel::DatabaseConnectionError
+                end
+
+                request_counter += 1
+
+              end.resume
+            end
+
+            db.pool.instance_eval { @pending.length }.must_equal 10
+
+            tick_sleep while request_counter < expected_runs
+
+            db.pool.instance_eval { @pending.length }.must_equal 0
+            db.pool.size.must_equal 0
+            request_counter.must_equal expected_runs
+          end
+
+        end
+      end
+    end
+
+    describe "play nice with transactions" do
+
+      let(:size) { 2 }
+
+      it "should lock connection to fiber" do
+        synchrony do
+          db.transaction do |conn|
+            db.in_transaction?.must_equal true
+            db.transaction do |inner_conn|
+              inner_conn.must_be_same_as conn
+              db.in_transaction?.must_equal true
+            end
+          end
+
+        end
+      end
+
+      it "should allow separate transactions" do
+        synchrony do
+          db.transaction do |conn|
+            db.in_transaction?.must_equal true
+            fiber_iterator.new([1,2], 2).each do |t|
+              db.in_transaction?.must_equal false
+              db.transaction do |inner_conn|
+                inner_conn.wont_be_same_as conn
+                db.in_transaction?.must_equal true
+              end
+            end
+          end
+
         end
       end
     end

data/spec/spec_helper.rb

@@ -16,6 +16,24 @@ DB_CONFIG = {
   password: "postgres",
 }
 
+module SynchronyUtils
+  def tick_sleep
+    f = Fiber.current
+    EM::next_tick { f.resume }
+    Fiber.yield
+  end
+
+  def synchrony
+    EM.synchrony do
+      begin
+        yield
+      ensure
+        EM.stop
+      end
+    end
+  end
+end
+
 DB_URL = "postgres://%s:%s@%s:%d/%s" % [DB_CONFIG[:user], DB_CONFIG[:password], DB_CONFIG[:host], DB_CONFIG[:port], DB_CONFIG[:dbname]]
 
 MiniTest::Reporters.use! MiniTest::Reporters::SpecReporter.new

metadata

@@ -1,80 +1,65 @@
 --- !ruby/object:Gem::Specification
 name: em-pg-sequel
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
 - Petr Yanovich
+- Rafal Michalski
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-25 00:00:00.000000000 Z
+date: 2014-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: em-synchrony
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
   type: :runtime
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
   name: em-pg-client
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.0
+    none: false
   type: :runtime
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.3.0
+    none: false
 - !ruby/object:Gem::Dependency
   name: em-synchrony
+  prerelease: false
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
   type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 description: Sequel adapter for ruby-em-pg-client
 email:
 - fl00r@yandex.ru
@@ -99,17 +84,17 @@ rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+  none: false
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24