my-sequel-synchrony 0.0.1

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in my-sequel-synchrony.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Ilya Maykov
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,75 @@
+# my-sequel-synchrony
+
+A fiber-aware MySQL adapter for [Sequel](https://github.com/jeremyevans/sequel) that works with
+[em-synchrony](https://github.com/igrigorik/em-synchrony).
+Uses the Mysql2::EM::Client client with the em-synchrony extensions.
+
+Note: this code is in an early stage of active development and has not been battle-tested in any
+production environments. For now, use at your own risk, and please report any bugs you find!
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'my-sequel-synchrony'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install my-sequel-synchrony
+
+## Usage
+
+Inside an `EM.synchrony` block or a synchrony-enabled web server (such as
+[sinatra-synchrony](https://github.com/kyledrake/sinatra-synchrony)), simply create the
+Sequel connection as usual, and specify the `:adapter => :mysql2synchrony` option:
+
+    @database = Sequel.connect(
+      :adapter => :mysql2synchrony,
+      :hostname => ...,
+      ...)
+
+And use it just like you would normally use a Sequel database object! If you have multiple DB shards
+(either master/slaves, or data sharded across multiple masters), you can specify them by passing in
+a `:servers` option just like you would with any other Sequel adapter that supports sharding (see
+the [Sequel sharding doc](https://github.com/jeremyevans/sequel/blob/master/doc/sharding.rdoc)).
+
+You can even run queries in parallel with `FiberIterator`:
+
+    require 'em-synchrony/fiber_iterator'
+
+    @shard_ids = [ :master, :slave1, :slave2 ]
+
+    # Assuming we're in a sinatra-synchrony webapp, this will run a simple "am i connected?" check
+    # on every shard in parallel and return 200 or 500 depending on the result.
+    get "/healthz" do
+      shard_errors = {}
+      EM::Synchrony::FiberIterator.new(@shard_ids, @shard_ids.size).each do |shard_id|
+        begin
+          @db.run("select 1;", :server => shard_id)
+        rescue Exception => e
+          shard_errors[shard_id] = e
+        end
+      end
+      halt 200, "All DB shards ok" if shard_errors.empty?
+
+      message = "#{shard_errors.size} DB shards had errors:\n"
+      message += shard_errors.map { |shard_id, error| "#{shard_id}: #{e.message}" }.join("\n")
+      halt 500, message
+    end
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Authors
+
+* Bo Chen (bochen.chen@gmail.com)
+* Ilya Maykov (ivmaykov@gmail.com)

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/lib/my-sequel-synchrony.rb

@@ -0,0 +1,3 @@
+require "my-sequel-synchrony/adapter"
+require "my-sequel-synchrony/connection_pool"
+require "my-sequel-synchrony/version"

data/lib/my-sequel-synchrony/adapter.rb

@@ -0,0 +1,55 @@
+# Sequel adapter for MySQL that uses the Mysql2::EM::Client and a fiber-aware connection pool.
+#
+# See also:
+#   https://github.com/jeremyevans/sequel/blob/master/lib/sequel/adapters/mysql2.rb
+#   https://github.com/brianmario/mysql2/blob/master/lib/mysql2/em.rb
+
+require "sequel"
+require "sequel/adapters/mysql2"
+require "em-synchrony/mysql2"
+require "my-sequel-synchrony/connection_pool"
+
+module Sequel::Mysql2::Synchrony
+  class Database < ::Sequel::Mysql2::Database
+    set_adapter_scheme :mysql2synchrony
+
+    def initialize(opts={})
+      pool_class = if opts.include?(:servers)
+        ::Sequel::Mysql2::Synchrony::ShardedConnectionPool
+      else
+        ::Sequel::Mysql2::Synchrony::ConnectionPool
+      end
+      super(opts.merge(:pool_class => pool_class, :connection_handling => :queue))
+    end
+
+    # Connect to the database. Similar to the Sequel::Mysql2#connect method, but uses the eventmachine
+    # client.
+    #
+    # NOTE: this code is mostly a copy of Sequel::Mysql2#connect() method, with the only difference being
+    # the connection class (Mysql::EM::Client instead of Mysql2::Client). This code could be removed if
+    # the parent class accepted a :use_eventmachine option.
+    def connect(server)
+      opts = server_opts(server)
+      opts[:host] ||= 'localhost'
+      opts[:username] ||= opts[:user]
+      opts[:flags] = ::Mysql2::Client::FOUND_ROWS if ::Mysql2::Client.const_defined?(:FOUND_ROWS)
+      conn = ::Mysql2::EM::Client.new(opts)
+      conn.query_options.merge!(:symbolize_keys=>true, :cache_rows=>false)
+
+      sqls = mysql_connection_setting_sqls
+
+      # Set encoding a slightly different way after connecting,
+      # in case the READ_DEFAULT_GROUP overrode the provided encoding.
+      # Doesn't work across implicit reconnects, but Sequel doesn't turn on
+      # that feature.
+      if encoding = opts[:encoding] || opts[:charset]
+        sqls.unshift("SET NAMES #{conn.escape(encoding.to_s)}")
+      end
+
+      sqls.each{|sql| log_yield(sql){conn.query(sql)}}
+
+      add_prepared_statements_cache(conn)
+      conn
+    end
+  end
+end

data/lib/my-sequel-synchrony/connection_pool.rb

@@ -0,0 +1,294 @@
+require "fiber"
+require "set"
+require "sequel"
+require "sequel/adapters/mysql2"
+
+# A Fiber-aware Sequel::ConnectionPool that works with EM::Synchrony! This version is not shard-aware,
+# that may be a TODO in the future.
+module Sequel::Mysql2::Synchrony
+  class ConnectionPool < ::Sequel::ConnectionPool
+    # The maximum number of connections this pool will create (per shard/server
+    # if sharding).
+    attr_reader :max_size
+
+    DEFAULT_MAX_SIZE = 4
+
+    # The following additional options are respected:
+    # * :connection_handling - Set how to handle available connections.  By default,
+    #   uses a a stack for performance.  Can be set to :queue to use a queue, which reduces
+    #   the chances of connections becoming stale. Can also be set to :disconnect, which will
+    #   disconnect after every query or transaction.
+    #
+    # * :max_connections - The maximum number of connections the connection pool
+    #   will open (default 4)
+    def initialize(opts = {}, &block)
+      super
+      @max_size = Integer(opts[:max_connections] || DEFAULT_MAX_SIZE)
+      raise(Sequel::Error, ':max_connections must be positive') if @max_size < 1
+      @connection_handling = opts[:connection_handling]
+      @available_connections = []
+      @waiting_fibers = []
+      @in_use_connections = {}
+      @pending_disconnects = Set.new
+      # TODO(ilyam): remove the debug prints once this is rock-solid and has good tests etc.
+      @debug = opts[:debug]
+    end
+
+    # The total number of open connections, either available or in-use.
+    def size
+      @in_use_connections.size + @available_connections.size
+    end
+
+    # Removes all connections currently available, optionally
+    # yielding each connection to the given block. This method has the effect of
+    # disconnecting from the database, assuming that no connections are currently
+    # being used. Schedules all in-use connections to be disconnected next time they are
+    # released into the available pool.
+    #
+    # After a disconnect, when connections are requested using #hold, the connection pool will
+    # create new connections to the database.
+    def disconnect(opts={}, &block)
+      if_debug? { dputs "disconnect(opts = #{opts.inspect})! Current fiber = #{Fiber.current.inspect}" }
+      connections_to_disconnect, @available_connections = @available_connections, []
+      @pending_disconnects += @in_use_connections.values.to_set
+      block ||= @disconnection_proc
+      connections_to_disconnect.each do |conn|
+        disconnect_conn(conn, opts, block)
+      end
+    end
+
+    # Chooses the first available connection, or if none are
+    # available, creates a new connection.  Passes the connection to the supplied
+    # block:
+    #
+    #   pool.hold {|conn| conn.execute('DROP TABLE posts')}
+    #
+    # Pool#hold is re-entrant, meaning it can be called recursively in
+    # the same fiber safely.
+    #
+    # If no connection is immediately available and the pool is already using the maximum
+    # number of connections, Pool#hold will block the current fiber until a connection
+    # is available.
+    def hold(server = nil)
+      if_debug? { dputs "hold(server = #{server.inspect})! Current fiber = #{Fiber.current.inspect}" }
+      fiber, conn = Fiber.current, nil
+      if conn = @in_use_connections[fiber]
+        return yield(conn)
+      end
+
+      begin
+        conn = acquire(fiber)
+        yield conn
+      rescue Sequel::DatabaseDisconnectError => error
+        disconnect_conn(conn) if conn
+        conn = nil
+        @in_use_connections.delete(fiber)
+        resume_next_waiting_fiber
+        raise error
+      ensure
+        release(fiber) if conn
+      end
+    end
+
+    private
+
+    # Assigns a connection to the given fiber. Suspends the fiber until a connection is available.
+    # Suspended fibers are served in FIFO order.
+    def acquire(fiber)
+      if_debug? { dputs "acquire(fiber = #{fiber.inspect})! Current fiber = #{Fiber.current.inspect}" }
+      conn = nil
+      while conn.nil?
+        if conn = available
+          @in_use_connections[fiber] = conn
+        else
+          wait_for_connection(fiber)
+        end
+      end
+      conn
+    end
+
+    # Releases the connection assigned to the given fiber back to the pool. May actually disconnect the
+    # connection in one of two cases:
+    #   1) A Pool#disconnect() was called while this connection was in use, which added it to the
+    #      @pending_disconnects set.
+    #   2) The pool was initialized with :connection_handling => :disconnect
+    def release(fiber)
+      if_debug? { dputs "release(fiber = #{fiber.inspect})! Current fiber = #{Fiber.current.inspect}" }
+      conn = @in_use_connections.delete(fiber)
+      raise(Sequel::Error, "release called on fiber that doesn't own a connection!") unless conn
+
+      if (@connection_handling == :disconnect || @pending_disconnects.include?(conn))
+        disconnect_conn(conn)
+      elsif @connection_handling == :queue
+        @available_connections.unshift(conn)
+      else
+        @available_connections << conn
+      end
+      resume_next_waiting_fiber
+    end
+
+    # Disconnects the connection and removes it from @pending_disconnects if it was in the set.
+    def disconnect_conn(conn, opts={}, &block)
+      if_debug? do
+        dputs "disconnect_conn(conn = #{conn.inspect}, opts=#{opts.inspect}, block = #{block.inspect})! "\
+              "Current fiber = #{Fiber.current.inspect}"
+      end
+      block ||= @disconnection_proc
+      block.call(conn) if block
+      @pending_disconnects.delete(conn)
+    end
+
+    # Suspends the given fiber until a connection becomes available.
+    def wait_for_connection(fiber)
+      if_debug? do
+        dputs "wait_for_connection(fiber = #{fiber.inspect})! Current fiber = #{Fiber.current.inspect}"
+      end
+      Fiber.yield @waiting_fibers.push fiber
+    end
+
+    # Resumes the next fiber waiting for a connection
+    def resume_next_waiting_fiber
+      if_debug? { dputs "resume_next_waiting_fiber()! Current fiber = #{Fiber.current.inspect}" }
+      if pending_fiber = @waiting_fibers.shift
+        if_debug? { dputs "  => pending_fiber = #{pending_fiber.inspect}" }
+        pending_fiber.resume
+      end
+    end
+
+    # Returns an available connection. If no connection is available, tries to create a new connection if the
+    # current pool size is smaller than the max pool size. May return nil.
+    def available
+      if_debug? { dputs "available()! Current fiber = #{Fiber.current.inspect}" }
+      @available_connections.pop || make_new(DEFAULT_SERVER)
+    end
+
+    # Alias the default make_new method, so subclasses can call it directly.
+    alias default_make_new make_new
+
+    # Creates a new connection to the given server if the size of the pool for
+    # the server is less than the maximum size of the pool.
+    def make_new(server)
+      if_debug? { dputs "make_new(server = #{server.inspect})! Current fiber = #{Fiber.current.inspect}" }
+      if size >= @max_size
+        dead_fibers = @in_use_connections.keys.reject { |fiber| fiber.alive? }
+        dead_fibers.each { |fiber| release(fiber) }
+      end
+      size < @max_size ? super(server) : nil
+    end
+
+    # Calls the given block if @debug is true
+    def if_debug?(&block)
+      block.call if @debug
+    end
+
+    # like puts, but goes to STDERR and prepends the string "DEBUG: "
+    def dputs(*args)
+      args.each { |arg| STDERR.puts "DEBUG: #{arg}" }
+      nil
+    end
+  end
+
+  # The sharded version of the em-synchrony connection pool is automatically used if a :servers option is
+  # passed to Sequel.connect(). It keeps a mapping from server ids to non-sharded connection pools, and
+  # delegates calls to them as needed.
+  class ShardedConnectionPool < ::Sequel::ConnectionPool
+    # The following additional options are respected:
+    # * :servers - A hash of servers to use.  Keys should be symbols.  If not
+    #   present, will use a single :default server.  The server name symbol will
+    #   be passed to the connection_proc.
+    #   The values in the hash should be per-server specific options (such as different :hostname or :database)
+    #
+    # * :servers_hash - The base hash to use for the servers.  By default,
+    #   Sequel uses Hash.new(:default).  You can use a hash with a default proc
+    #   that raises an error if you want to catch all cases where a nonexistent
+    #   server is used.
+    def initialize(opts = {}, &block)
+      # NOTE: @initialize_opts is passed to the internal non-sharded connection pools when they are created.
+      @initialize_opts = opts.dup
+      @initialize_opts.delete(:servers)
+      @initialize_opts.delete(:servers_hash)
+      @initialize_block = block
+
+      @server_ids = opts[:servers_hash] || Hash.new(:default)
+      @server_pools = {}
+      add_servers([:default] + (opts[:servers] ? opts[:servers].keys : []))
+    end
+
+    # Adds new servers to the connection pool. Primarily used in conjunction with master/slave
+    # or shard configurations. Allows for dynamic expansion of the potential slaves/shards
+    # at runtime. servers argument should be an array of symbols.
+    def add_servers(servers)
+      servers.each do |server|
+        next if @server_ids.has_key?(server)
+
+        @server_ids[server] = server
+        @server_pools[server] = ::Sequel::Mysql2::Synchrony::ConnectionPool.new(@initialize_opts) do
+          @initialize_block.call(server) if @initialize_block
+        end
+      end
+    end
+
+    # Remove servers from the connection pool. Primarily used in conjunction with master/slave
+    # or shard configurations.  Similar to disconnecting from all given servers,
+    # except that after it is used, future requests for the server will use the
+    # :default server instead.
+    def remove_servers(servers)
+      raise(Sequel::Error, "cannot remove default server") if servers.include?(:default)
+      pools_to_disconnect = []
+      servers.each do |server|
+        pool = @server_pools.delete(server)
+        pools_to_disconnect << pool if pool
+      end
+      pools_to_disconnect.each { |pool| pool.disconnect }
+    end
+
+    # The total number of connections opened for the given server, should
+    # be equal to available_connections.length + allocated.length.  Nonexistent
+    # servers will return the created count of the default server.
+    def size(server = :default)
+      @server_pools[@server_ids[server]].size
+    end
+
+    # Removes all connections currently available on all servers, optionally
+    # yielding each connection to the given block. This method has the effect of
+    # disconnecting from the database, assuming that no connections are currently
+    # being used.  If connections are being used, they are scheduled to be
+    # disconnected as soon as they are returned to the pool.
+    #
+    # Once a connection is requested using #hold, the connection pool
+    # creates new connections to the database. Options:
+    # * :server - Should be a symbol specifing the server to disconnect from,
+    #   or an array of symbols to specify multiple servers. If not specified,
+    #   then all servers are disconnected.
+    def disconnect(opts = {}, &block)
+      servers_to_disconnect = @server_pools.keys
+      servers_to_disconnect &= Array(opts[:servers]) if opts[:servers]
+      servers_to_disconnect.each do |server_id|
+        @server_pools[server_id].disconnect(opts, &block)
+      end
+    end
+
+    # Chooses the first available connection to the given server, or if none are
+    # available, creates a new connection.  Passes the connection to the supplied
+    # block:
+    #
+    #   pool.hold {|conn| conn.execute('DROP TABLE posts')}
+    #
+    # Pool#hold is re-entrant, meaning it can be called recursively in
+    # the same thread without blocking.
+    #
+    # If no connection is immediately available and the pool is already using the maximum
+    # number of connections, Pool#hold will block until a connection
+    # is available or the timeout expires.  If the timeout expires before a
+    # connection can be acquired, a Sequel::PoolTimeout is
+    # raised.
+    def hold(server = :default, &block)
+      @server_pools[@server_ids[server]].hold(nil, &block)
+    end
+
+    # Return an array of symbols for servers in the connection pool.
+    def servers
+      @server_ids.keys
+    end
+  end
+end

data/lib/my-sequel-synchrony/version.rb

@@ -0,0 +1,7 @@
+module Sequel
+  module Mysql2
+    module Synchrony
+      VERSION = "0.0.1"
+    end
+  end
+end

data/my-sequel-synchrony.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/my-sequel-synchrony/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = [ "Ilya Maykov" ]
+  gem.email         = [ "ivmaykov@gmail.com" ]
+  gem.description   = "A fiber-aware MySQL adapter for Sequel that works with em-synchrony"
+  gem.summary       = "A fiber-aware MySQL adapter for Sequel that works with em-synchrony"
+  gem.homepage      = "https://github.com/ivmaykov/my-sequel-synchrony"
+
+  gem.files         = `git ls-files`.split($\)
+  # gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  # gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "my-sequel-synchrony"
+  gem.require_paths = [ "lib" ]
+  gem.version       = Sequel::Mysql2::Synchrony::VERSION
+  gem.required_ruby_version = " >= 1.9.2"
+
+  gem.add_dependency("sequel", "~> 3.39.0")
+  gem.add_dependency("mysql2", "~> 0.3.11")
+  gem.add_dependency("em-synchrony", "~> 1.0.0")
+  gem.add_dependency("rake", "~> 0.8.7")
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: my-sequel-synchrony
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Ilya Maykov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.39.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.39.0
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.11
+- !ruby/object:Gem::Dependency
+  name: em-synchrony
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.7
+description: A fiber-aware MySQL adapter for Sequel that works with em-synchrony
+email:
+- ivmaykov@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/my-sequel-synchrony.rb
+- lib/my-sequel-synchrony/adapter.rb
+- lib/my-sequel-synchrony/connection_pool.rb
+- lib/my-sequel-synchrony/version.rb
+- my-sequel-synchrony.gemspec
+homepage: https://github.com/ivmaykov/my-sequel-synchrony
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: A fiber-aware MySQL adapter for Sequel that works with em-synchrony
+test_files: []