em_postgresql 0.3.0

data/README.md

@@ -0,0 +1,43 @@
+em_postgresql
+---------------
+
+An EventMachine-aware driver for using Postgresql with ActiveRecord.
+
+Requirements
+==============
+
+* Ruby 1.9
+* EventMachine 0.12.10
+* postgres-pr 0.6.1
+* Rails 2.3.5
+
+Tested with these version, other versions might work.  YMMV.
+
+You CANNOT have the **pg** gem installed.  ActiveRecord prefers the **pg** gem but this code requires
+the **postgres-pr** gem to be loaded.  I'm not sure if there is a way to make them live together in harmony.
+
+You'll need to ensure your code is running within an active Fiber using the FiberPool defined in fiber_pool.rb.  If you are running Rails in Thin, the following code is a good place to start to figure out how to do this:
+
+<http://github.com/espace/neverblock/blob/master/lib/never_block/servers/thin.rb>
+
+Usage
+=======
+
+List this gem in your `config/environment.rb`:
+
+    config.gem 'postgres-pr', :lib => false
+    config.gem 'em_postgresql', :lib => false
+
+and update your `config/database.yml` to contain the proper adapter attribute:
+
+    adapter: em_postgresql
+
+
+Author
+=========
+
+Mike Perham, mperham AT gmail.com,
+[Github](http://github.com/mperham), 
+[Twitter](http://twitter.com/mperham),
+[Blog](http://mikeperham.com)
+

data/Rakefile

@@ -0,0 +1,31 @@
+# vim: syntax=Ruby
+require 'rubygems'
+require 'rake/testtask'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "em_postgresql"
+    s.summary = s.description = "An ActiveRecord driver for using Postgresql with EventMachine"
+    s.email = "mperham@gmail.com"
+    s.homepage = "http://github.com/mperham/em_postgresql"
+    s.authors = ['Mike Perham']
+    s.files = FileList["[A-Z]*", "{lib,test}/**/*"]
+    s.test_files = FileList["test/test_*.rb"]
+    s.add_dependency 'postgres-pr', '>=0.6.1'
+    s.add_dependency 'eventmachine', '>=0.12.10'
+  end
+
+rescue LoadError
+  puts "Jeweler not available. Install it for jeweler-related tasks with: sudo gem install jeweler"
+end
+
+
+task :gin => [:gemspec, :build] do
+	puts `gem install pkg/em_postgresql-#{File.read('VERSION').strip}.gem`
+end
+
+
+Rake::TestTask.new
+
+task :default => :test

data/VERSION

@@ -0,0 +1 @@
+0.3.0

data/lib/active_record/connection_adapters/em_postgresql_adapter.rb

@@ -0,0 +1,116 @@
+require 'postgres_connection'
+
+require 'active_record'
+require 'active_record/connection_adapters/postgresql_adapter'
+require 'active_record/patches'
+
+if !PGconn.respond_to?(:quote_ident)
+  def PGconn.quote_ident(name)
+    %("#{name}")
+  end
+end
+
+module ActiveRecord
+  module ConnectionAdapters
+
+    class EmPostgreSQLAdapter < PostgreSQLAdapter
+      # checkin :logi
+      # checkout :logo
+      # 
+      # def logo
+      #   puts "#{Fiber.current.object_id} #{self.object_id} checkout"
+      # end
+      # def logi
+      #   puts "#{Fiber.current.object_id} #{self.object_id} checkin"
+      # end
+
+      def initialize(connection, logger, host_parameters, connection_parameters, config)
+        @hostname = host_parameters[0]
+        @port = host_parameters[1]
+        @connect_parameters, @config = connection_parameters, config
+        super(connection, logger, nil, config)
+      end
+      
+      def connect
+        @logger.info "Connecting to #{@hostname}:#{@port}"
+        @connection = ::EM.connect(@hostname, @port, ::EM::P::PostgresConnection)
+
+        fiber = Fiber.current
+        yielding = true
+        result = false
+        message = nil
+        task = @connection.connect(*@connect_parameters)
+        task.callback do |rc, msg|
+          result = rc
+          message = msg
+          fiber.resume
+        end
+        task.errback do |msg|
+          result = false
+          message = msg
+          yielding = false
+        end
+        Fiber.yield if yielding
+
+        raise RuntimeError, "Connection failed: #{message}" if !result
+        
+        # Use escape string syntax if available. We cannot do this lazily when encountering
+        # the first string, because that could then break any transactions in progress.
+        # See: http://www.postgresql.org/docs/current/static/runtime-config-compatible.html
+        # If PostgreSQL doesn't know the standard_conforming_strings parameter then it doesn't
+        # support escape string syntax. Don't override the inherited quoted_string_prefix.
+        if supports_standard_conforming_strings?
+          self.class.instance_eval do
+            define_method(:quoted_string_prefix) { 'E' }
+          end
+        end
+
+        # Money type has a fixed precision of 10 in PostgreSQL 8.2 and below, and as of
+        # PostgreSQL 8.3 it has a fixed precision of 19. PostgreSQLColumn.extract_precision
+        # should know about this but can't detect it there, so deal with it here.
+        money_precision = (postgresql_version >= 80300) ? 19 : 10
+        PostgreSQLColumn.module_eval(<<-end_eval)
+          def extract_precision(sql_type)  # def extract_precision(sql_type)
+            if sql_type =~ /^money$/       #   if sql_type =~ /^money$/
+              #{money_precision}           #     19
+            else                           #   else
+              super                        #     super
+            end                            #   end
+          end                              # end
+        end_eval
+
+        configure_connection
+        @connection
+      end
+      
+      def active?
+        !@connection.closed? && @connection.exec('SELECT 1')
+      rescue RuntimeError => re
+        false
+      end
+
+    end
+  end
+
+  class Base
+    # Establishes a connection to the database that's used by all Active Record objects
+    def self.em_postgresql_connection(config) # :nodoc:
+      config = config.symbolize_keys
+      host     = config[:host]
+      port     = config[:port] || 5432
+      username = config[:username].to_s if config[:username]
+      password = config[:password].to_s if config[:password]
+
+      if config.has_key?(:database)
+        database = config[:database]
+      else
+        raise ArgumentError, "No database specified. Missing argument: database."
+      end
+
+      # The postgres drivers don't allow the creation of an unconnected PGconn object,
+      # so just pass a nil connection object for the time being.
+      ConnectionAdapters::EmPostgreSQLAdapter.new(nil, logger, [host, port], [database, username, password], config)
+    end
+  end
+  
+end

data/lib/active_record/patches.rb

@@ -0,0 +1,102 @@
+module ActiveRecord
+  module ConnectionAdapters
+    
+    def self.fiber_pools
+      @fiber_pools ||= []
+    end
+    def self.register_fiber_pool(fp)
+      fiber_pools << fp
+    end
+
+    class FiberedMonitor
+      class Queue
+        def initialize
+          @queue = []
+        end
+
+        def wait(timeout)
+          t = timeout || 5
+          fiber = Fiber.current
+          x = EM::Timer.new(t) do
+            @queue.delete(fiber)
+            fiber.resume(false)
+          end
+          @queue << fiber
+          returning Fiber.yield do
+            x.cancel
+          end
+        end
+
+        def signal
+          fiber = @queue.pop
+          fiber.resume(true) if fiber
+        end
+      end
+      
+      def synchronize
+        yield
+      end
+
+      def new_cond
+        Queue.new
+      end
+    end
+    
+    # ActiveRecord's connection pool is based on threads.  Since we are working
+    # with EM and a single thread, multiple fiber design, we need to provide 
+    # our own connection pool that keys off of Fiber.current so that different
+    # fibers running in the same thread don't try to use the same connection.
+    class ConnectionPool
+      def initialize(spec)
+        @spec = spec
+
+        # The cache of reserved connections mapped to threads
+        @reserved_connections = {}
+
+        # The mutex used to synchronize pool access
+        @connection_mutex = FiberedMonitor.new
+        @queue = @connection_mutex.new_cond
+
+        # default 5 second timeout unless on ruby 1.9
+        @timeout = spec.config[:wait_timeout] || 5
+
+        # default max pool size to 5
+        @size = (spec.config[:pool] && spec.config[:pool].to_i) || 5
+
+        @connections = []
+        @checked_out = []
+      end
+
+      private
+
+      def current_connection_id #:nodoc:
+        Fiber.current.object_id
+      end
+
+      # Remove stale fibers from the cache.
+      def remove_stale_cached_threads!(cache, &block)
+        keys = Set.new(cache.keys)
+
+        ActiveRecord::ConnectionAdapters.fiber_pools.each do |pool|
+          pool.busy_fibers.each_pair do |object_id, fiber|
+            keys.delete(object_id)
+          end
+        end
+#        puts "Pruning stale connections: #{f.busy_fibers.size} #{f.fibers.size} #{keys.inspect}"
+        keys.each do |key|
+          next unless cache.has_key?(key)
+          block.call(key, cache[key])
+          cache.delete(key)
+        end
+      end
+
+      def checkout_and_verify(c)
+        @checked_out << c
+        c.run_callbacks :checkout
+        c.verify!
+        c
+      end
+    end
+
+  end
+end

data/lib/fiber_pool.rb

@@ -0,0 +1,82 @@
+# Author::    Mohammad A. Ali  (mailto:oldmoe@gmail.com)
+# Copyright:: Copyright (c) 2008 eSpace, Inc.
+# License::   Distributes under the same terms as Ruby
+
+require 'fiber'
+
+class Fiber
+  
+  #Attribute Reference--Returns the value of a fiber-local variable, using
+  #either a symbol or a string name. If the specified variable does not exist,
+  #returns nil.
+  def [](key)
+    local_fiber_variables[key]
+  end
+  
+  #Attribute Assignment--Sets or creates the value of a fiber-local variable,
+  #using either a symbol or a string. See also Fiber#[].
+  def []=(key,value)
+    local_fiber_variables[key] = value
+  end
+  
+  private
+  
+  def local_fiber_variables
+    @local_fiber_variables ||= {}
+  end
+end
+
+class FiberPool
+
+  # gives access to the currently free fibers
+  attr_reader :fibers
+  attr_reader :busy_fibers
+
+  # Code can register a proc with this FiberPool to be called
+  # every time a Fiber is finished.  Good for releasing resources
+  # like ActiveRecord database connections.
+  attr_accessor :generic_callbacks
+
+  # Prepare a list of fibers that are able to run different blocks of code
+  # every time. Once a fiber is done with its block, it attempts to fetch
+  # another one from the queue
+  def initialize(count = 100)
+    @fibers,@busy_fibers,@queue,@generic_callbacks = [],{},[],[]
+    count.times do |i|
+      fiber = Fiber.new do |block|
+        loop do
+          block.call
+          # callbacks are called in a reverse order, much like c++ destructor
+          Fiber.current[:callbacks].pop.call while Fiber.current[:callbacks].length > 0
+          generic_callbacks.each do |cb|
+            cb.call
+          end
+          unless @queue.empty?
+            block = @queue.shift
+          else
+            @busy_fibers.delete(Fiber.current.object_id)
+            @fibers << Fiber.current
+            block = Fiber.yield
+          end
+        end
+      end
+      fiber[:callbacks] = []
+      fiber[:em_keys] = []
+      @fibers << fiber
+    end
+  end
+
+  # If there is an available fiber use it, otherwise, leave it to linger
+  # in a queue
+  def spawn(&block)
+    if fiber = @fibers.shift
+      fiber[:callbacks] = []
+      @busy_fibers[fiber.object_id] = fiber
+      fiber.resume(block)
+    else
+      @queue << block
+    end
+    self # we are keen on hiding our queue
+  end
+
+end

data/lib/postgres_connection.rb

@@ -0,0 +1,209 @@
+require 'eventmachine'
+require 'postgres-pr/message'
+require 'postgres-pr/connection'
+require 'stringio'
+require 'fiber'
+
+class StringIO # :nodoc:
+  # Reads exactly +n+ bytes.
+  #
+  # If the data read is nil an EOFError is raised.
+  #
+  # If the data read is too short a TruncatedDataError is raised and the read
+  # data is obtainable via its #data method.
+  def readbytes(n)
+    str = read(n)
+    if str == nil
+      raise EOFError, "End of file reached"
+    end
+    if str.size < n
+      raise TruncatedDataError.new("data truncated", str) 
+    end
+    str
+  end
+  alias read_exactly_n_bytes readbytes
+end
+
+
+module EventMachine
+  module Protocols
+    class PostgresConnection < EventMachine::Connection
+      include PostgresPR
+
+      def initialize
+        @data = ""
+        @params = {}
+        @connected = false
+      end
+      
+      # Fibered impl for synchronous execution of SQL within EM
+      def exec(sql)
+        fiber = Fiber.current
+#        p [fiber.object_id, self.object_id, sql]
+        yielding = true
+        (status, result, errors) = nil
+        d = query(sql)
+        d.callback do |s, r, e|
+          (status, result, errors) = s, r, e
+          fiber.resume
+        end
+        d.errback do |msg|
+          errors = msg
+          status = false
+          # errback is called from the same fiber
+          yielding = false
+        end
+        
+        Fiber.yield if yielding
+#        p [fiber.object_id, self.object_id, result]
+        return PGresult.new(result) if status
+        raise RuntimeError, (errors || result).inspect
+      end
+
+      def close
+        close_connection
+      end
+
+      def closed?
+        !@connected
+      end
+      
+      def post_init
+        @connected = true
+      end
+
+      def unbind
+        @connected = false
+        if o = (@pending_query || @pending_conn)
+          o.succeed false, "lost connection"
+        end
+      end
+
+      def connect(db, user, psw=nil)
+        d = EM::DefaultDeferrable.new
+        d.timeout 15
+
+        if @pending_query || @pending_conn
+          d.fail "Operation already in progress"
+        else
+          @pending_conn = d
+          prms = {"user"=>user, "database"=>db}
+          @user = user
+          if psw
+            @password = psw
+            #prms["password"] = psw
+          end
+          send_data PostgresPR::StartupMessage.new( 3 << 16, prms ).dump
+        end
+
+        d
+      end
+
+      def query(sql)
+        d = EM::DefaultDeferrable.new
+        d.timeout 15
+
+        if !@connected
+          d.fail "Not connected"
+        elsif @pending_query || @pending_conn
+          d.fail "Operation already in progress"
+        else
+          @r = PostgresPR::Connection::Result.new
+          @e = []
+          @pending_query = d
+          send_data PostgresPR::Query.dump(sql)
+        end
+
+        d
+      end
+
+      def receive_data(data)
+        @data << data
+        while @data.length >= 5
+          pktlen = @data[1...5].unpack("N").first
+          if @data.length >= (1 + pktlen)
+            pkt = @data.slice!(0...(1+pktlen))
+            m = StringIO.open( pkt, "r" ) {|io| PostgresPR::Message.read( io ) }
+            if @pending_conn
+              dispatch_conn_message m
+            elsif @pending_query
+              dispatch_query_message m
+            else
+              raise "Unexpected message from database"
+            end
+          else
+            break # very important, break out of the while
+          end
+        end
+      end
+
+      # Cloned and modified from the postgres-pr.
+      def dispatch_conn_message(msg)
+        case msg
+        when AuthentificationClearTextPassword
+          raise ArgumentError, "no password specified" if @password.nil?
+          send_data PasswordMessage.new(@password).dump
+
+        when AuthentificationCryptPassword
+          raise ArgumentError, "no password specified" if @password.nil?
+          send_data PasswordMessage.new(@password.crypt(msg.salt)).dump
+
+        when AuthentificationMD5Password
+          raise ArgumentError, "no password specified" if @password.nil?
+          require 'digest/md5'
+
+          m = Digest::MD5.hexdigest(@password + @user)
+          m = Digest::MD5.hexdigest(m + msg.salt)
+          m = 'md5' + m
+          send_data PasswordMessage.new(m).dump
+
+        when AuthentificationKerberosV4, AuthentificationKerberosV5, AuthentificationSCMCredential
+          raise "unsupported authentification"
+
+        when AuthentificationOk
+        when ErrorResponse
+          raise msg.field_values.join("\t")
+        when NoticeResponse
+          @notice_processor.call(msg) if @notice_processor
+        when ParameterStatus
+          @params[msg.key] = msg.value
+        when BackendKeyData
+          # TODO
+          #p msg
+        when ReadyForQuery
+          # TODO: use transaction status
+          pc,@pending_conn = @pending_conn,nil
+          pc.succeed true
+        else
+          raise "unhandled message type"
+        end
+      end
+
+      # Cloned and modified from the postgres-pr.
+      def dispatch_query_message(msg)
+        case msg
+        when DataRow
+          @r.rows << msg.columns
+        when CommandComplete
+          @r.cmd_tag = msg.cmd_tag
+        when ReadyForQuery
+          pq,@pending_query = @pending_query,nil
+          pq.succeed @e.size == 0, @r, @e
+        when RowDescription
+          @r.fields = msg.fields
+        when CopyInResponse
+        when CopyOutResponse
+        when EmptyQueryResponse
+        when ErrorResponse
+          @e << msg.field_values[2]
+        when NoticeResponse
+          @notice_processor.call(msg) if @notice_processor
+        when ParameterStatus
+        else
+          # TODO
+          puts "Unknown Postgres message: #{msg}"
+        end
+      end
+    end
+  end
+end

data/test/database.yml

@@ -0,0 +1,10 @@
+test:
+  adapter: em_postgresql
+  host: localhost
+  username: mike
+  password: password
+  database: onespot_test
+  schema_search_path: bdu,public
+  statement_timeout: 60
+  encoding: UTF8
+  port: 5432

data/test/test_database.rb

@@ -0,0 +1,51 @@
+require 'rubygems'
+require 'logger'
+require 'yaml'
+require 'erb'
+
+gem 'activerecord', '>= 2.3.5'
+require 'active_record'
+
+RAILS_ENV='test'
+
+ActiveRecord::Base.configurations = YAML::load(ERB.new(File.read(File.join(File.dirname(__FILE__), 'database.yml'))).result)
+ActiveRecord::Base.default_timezone = :utc
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+ActiveRecord::Base.logger.level = Logger::INFO
+ActiveRecord::Base.pluralize_table_names = false
+ActiveRecord::Base.time_zone_aware_attributes = true
+Time.zone = 'UTC'
+
+require 'eventmachine'
+require 'test/unit'
+
+class Site < ActiveRecord::Base
+  set_table_name 'site'
+end
+
+class TestDatabase < Test::Unit::TestCase
+  def test_live_server
+    EM.run do
+      Fiber.new do
+        ActiveRecord::Base.establish_connection
+
+        result = ActiveRecord::Base.connection.query('select id, domain_name from site')
+        assert result
+        assert_equal 3, result.size
+
+        result = Site.all
+        assert result
+        assert_equal 3, result.size
+
+        result = Site.find(1)
+        assert_equal 1, result.id
+        assert_equal 'somedomain.com', result.domain_name
+      end.resume
+
+      EM.add_timer(1) do
+        EM.stop
+      end
+
+    end
+  end
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification 
+name: em_postgresql
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 3
+  - 0
+  version: 0.3.0
+platform: ruby
+authors: 
+- Mike Perham
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-04-03 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: postgres-pr
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 6
+        - 1
+        version: 0.6.1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 12
+        - 10
+        version: 0.12.10
+  type: :runtime
+  version_requirements: *id002
+description: An ActiveRecord driver for using Postgresql with EventMachine
+email: mperham@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+files: 
+- README.md
+- Rakefile
+- VERSION
+- lib/active_record/connection_adapters/em_postgresql_adapter.rb
+- lib/active_record/patches.rb
+- lib/fiber_pool.rb
+- lib/postgres_connection.rb
+- test/database.yml
+- test/test_database.rb
+has_rdoc: true
+homepage: http://github.com/mperham/em_postgresql
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: An ActiveRecord driver for using Postgresql with EventMachine
+test_files: 
+- test/test_database.rb