em-postgres 0.0.1

data/README

@@ -0,0 +1,12 @@
+Async PostgreSQL driver for Ruby/EventMachine
+  based off Aman Gupta's mysql driver
+
+
+Requires pg
+
+
+
+TODOS:
+get a working java driver
+ability to use different MRI postgres drivers (postgres-pr, pg, postgres)
+add working adaptors for activerecord and sequel

data/Rakefile

@@ -0,0 +1,19 @@
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "em-postgres"
+    gemspec.summary = "Async PostgreSQL driver for Ruby/Eventmachine"
+    gemspec.description = gemspec.summary
+    gemspec.email = "jtoy@jtoy.net"
+    gemspec.homepage = "http://github.com/jtoy/em-postgres"
+    gemspec.authors = ["Jason Toy"]
+    gemspec.add_dependency('eventmachine', '>= 0.12.9')
+    gemspec.rubyforge_project = "em-postgres"
+  end
+
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end

data/em-postgres.gemspec

@@ -0,0 +1,24 @@
+spec = Gem::Specification.new do |s|
+  s.name = 'em-postgres'
+  s.version = '0.0.1'
+  s.date = '2011-08-25'
+  s.summary = 'Async PostgreSQL client API for Ruby/EventMachine'
+  s.email = "jtoy@jtoy.net"
+  s.homepage = "http://github.com/jtoy/em-postgres"
+  s.description = 'Async PostgreSQL client API for Ruby/EventMachine'
+  s.has_rdoc = false
+  s.authors = ["Jason Toy"]
+  s.add_dependency('eventmachine', '>= 0.12.9')
+
+  # git ls-files
+  s.files = %w[
+    README
+    Rakefile
+    em-postgres.gemspec
+    lib/em-postgres/postgres.rb
+    lib/em-postgres/connection.rb
+    lib/em-postgres.rb
+    spec/helper.rb
+    spec/postgres_spec.rb
+  ]
+end

data/lib/em-postgres.rb

@@ -0,0 +1,7 @@
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+
+require "eventmachine"
+
+%w[ postgres connection ].each do |file|
+  require "em-postgres/#{file}"
+end

data/lib/em-postgres/connection.rb

@@ -0,0 +1,224 @@
+
+class Postgres
+  def result
+    @cur_result
+  end
+end
+
+module EventMachine
+  class PostgresConnection < EventMachine::Connection
+
+    attr_reader :processing, :connected, :opts
+    alias :settings :opts
+
+    MAX_RETRIES_ON_DEADLOCKS = 10
+
+    DisconnectErrors = [
+      'query: not connected',
+      'Postgres server has gone away',
+      'Lost connection to Postgres server during query'
+    ] unless defined? DisconnectErrors
+
+    def initialize(postgres,opts,conn) 
+    #def initialize(postgres,opts) 
+      
+      begin
+      @conn = conn
+      @postgres = postgres
+      @fd = postgres.socket
+      @opts = opts
+      @current = nil
+      @queue = []
+      @processing = false
+      @connected = true
+
+      self.notify_readable = true
+      EM.add_timer(0){ next_query }
+    rescue => e
+      puts e.inspect
+    end
+    end
+    
+    def self.connect(opts)
+      if conn = connect_socket(opts)
+        #debug [:connect, conn.socket, opts]
+        EM.watch(conn.socket, EventMachine::PostgresConnection,conn,opts)
+      else
+        # invokes :errback callback in opts before firing again
+        debug [:reconnect]
+        EM.add_timer(5) { connect opts }
+      end
+    end
+    
+    # stolen from sequel
+     def self.connect_socket(opts)
+     begin
+       conn = PGconn.connect(
+         opts[:host],
+         (opts[:port]), #TODO deal with host and port
+         nil,nil,
+         opts[:database],
+         opts[:user],
+         opts[:password]
+       )
+       # set encoding _before_ connecting
+       if encoding = opts[:encoding] || opts[:charset]
+         if conn.respond_to?(:set_client_encoding)
+           conn.set_client_encoding(encoding)
+         else
+           conn.async_exec("set client_encoding to '#{encoding}'")
+         end
+       end
+       conn
+     rescue Exception => e
+       puts "#{e} exception"
+       if cb = opts[:errback]
+         cb.call(e)
+         nil
+       else
+         raise e
+       end
+     end
+     end
+
+    def notify_readable
+      if item = @current
+        sql, cblk, eblk, retries = item
+        #results = []
+        #result = nil
+        #@postgres.get_result{|r| result = r}
+        #@postgres.get_result #TODO remove this, I can't process anymore code without this.
+        result = nil        
+        loop do
+          # Fetch the next result. If there isn't one, the query is 
+          # finished
+          item = @postgres.get_result
+          if item
+            result = item
+          else
+            break
+          end
+          #puts "\n\nQuery result:\n%p\n" % [ result.values ]
+        end
+        
+        unless @postgres.error_message == ""
+          #TODO this is wrong
+          eb = (eblk || @opts[:on_error])
+          eb.call(result) if eb
+          result.clear
+          #reconnect
+          @processing = false
+          #@current = nil
+          return next_query
+        end
+        # kick off next query in the background
+        # as we process the current results
+        @current = nil
+        @processing = false
+        cblk.call(result) if cblk
+        result.clear
+        next_query
+      else
+        return close
+      end
+
+    rescue Exception => e
+      puts "error #{e}"
+      if e.message =~ /Deadlock/ and retries < MAX_RETRIES_ON_DEADLOCKS
+        @queue << [sql, cblk, eblk, retries + 1]
+        @processing = false
+        next_query
+
+      elsif DisconnectErrors.include? e.message
+        @queue << [sql, cblk, eblk, retries + 1]
+        return #close
+
+      elsif cb = (eblk || @opts[:on_error])
+        cb.call(e)
+        @processing = false
+        next_query
+
+      else
+        raise e
+      end
+    end
+    
+    def unbind
+
+      # wait for the next tick until the current fd is removed completely from the reactor
+      #
+      # in certain cases the new FD# (@mysql.socket) is the same as the old, since FDs are re-used
+      # without next_tick in these cases, unbind will get fired on the newly attached signature as well
+      #
+      # do _NOT_ use EM.next_tick here. if a bunch of sockets disconnect at the same time, we want
+      # reconnects to happen after all the unbinds have been processed
+
+      #@connected = false
+      EM.next_tick { reconnect }
+    end
+    
+    def reconnect
+      puts "DDDDD"
+      @processing = false
+      @postgres = @conn.connect_socket(@opts)
+      @fd = @postgres.socket
+
+      @signature = EM.attach_fd(@postgres.socket, true)
+      EM.set_notify_readable(@signature, true)
+      EM.instance_variable_get('@conns')[@signature] = self
+      @connected = true
+      next_query
+
+    rescue Exception => e
+      EM.add_timer(1) { reconnect }
+    end
+
+
+    def execute(sql, cblk = nil, eblk = nil, retries = 0)
+      
+      begin
+        if not @processing or not @connected
+        #if !@processing || !@connected
+          @processing = true
+
+          @postgres.send_query(sql)          
+        else          
+          @queue << [sql, cblk, eblk, retries]
+          return
+        end
+
+      rescue Exception => e
+        puts "error in execute #{e}"
+        if DisconnectErrors.include? e.message
+          @queue << [sql, cblk, eblk, retries]
+          return #close
+        else
+          raise e
+        end
+      end
+      @current = [sql, cblk, eblk, retries]
+    end
+
+    # act like the pg driver
+    def method_missing(method, *args, &blk)
+      @postgres.send(method, *args, &blk) if @postres.respond_to? method
+    end
+
+    def close
+      return unless @connected
+      detach
+      @postgres.finish
+      @connected = false
+    end
+
+    private
+
+      def next_query
+        if @connected and !@processing and pending = @queue.shift
+          sql, cblk, eblk = pending
+          execute(sql, cblk, eblk)
+        end
+      end
+
+  end
+end

data/lib/em-postgres/postgres.rb

@@ -0,0 +1,143 @@
+require "eventmachine"
+require "pg"
+require "fcntl"
+
+=begin
+module EventMachine
+  class Postgres
+
+  def self.settings
+    @settings ||= { :connections => 1, :logging => false,:database=>"test" }
+  end
+
+  def self.execute query, cblk = nil, eblk = nil, &blk
+    @n ||= 0
+    connection = connection_pool[@n]
+    @n = 0 if (@n+=1) >= connection_pool.size
+
+    #connection.execute(query, type, cblk, eblk, &blk)
+    
+    df = EventMachine::DefaultDeferrable.new
+    cb = blk || Proc.new { |r| df.succeed(r) }
+    eb = Proc.new { |r| df.fail(r) }
+    connection.execute(query,cb, eb)
+    df
+  end
+  #class << self
+  #  alias query execute 
+  #end
+  def self.connection_pool
+    @connection_pool ||= (1..settings[:connections]).map{ EventMachine::PostgresConnection.connect(settings) }
+    
+  end
+end
+end
+=end
+
+
+module EventMachine
+  class Postgres
+
+    #self::Postgres = ::Postgres unless defined? self::Postgres
+
+    attr_reader :connection
+
+    def initialize(opts)
+      unless EM.respond_to?(:watch) and PGconn.method_defined?(:socket)
+        
+        raise RuntimeError, 'pg and EM.watch are required for EventedPostgres'
+      end
+
+      @settings = { :debug => false }.merge!(opts)
+      @connection = connect(@settings)
+    end
+
+    def close
+      @connection.close
+    end
+
+    def query(sql, &blk)
+      df = EventMachine::DefaultDeferrable.new
+      cb = blk || Proc.new { |r| df.succeed(r) }
+      eb = Proc.new { |r| df.fail(r) }
+
+      @connection.execute(sql, cb, eb)
+
+      df
+    end
+    alias :real_query :query
+    alias :execute :query
+    # behave as a normal postgres connection
+    def method_missing(method, *args, &blk)
+      @connection.send(method, *args)
+    end
+
+    def connect(opts)
+      if conn = connect_socket(opts)
+        #debug [:connect, conn.socket, opts]
+        #EM.watch(conn.socket, EventMachine::PostgresConnection, conn, opts, self)
+
+        EM.watch(conn.socket, EventMachine::PostgresConnection,conn,opts,self)
+      else
+        # invokes :errback callback in opts before firing again
+        debug [:reconnect]
+        EM.add_timer(5) { connect opts }
+      end
+    end
+
+    # stolen from sequel
+    def connect_socket(opts)
+    begin
+      conn = PGconn.connect(
+        opts[:host],
+        (opts[:port]), #TODO deal with host and port
+        nil,nil,
+        opts[:database],
+        opts[:user],
+        opts[:password]
+      )
+      # set encoding _before_ connecting
+      if encoding = opts[:encoding] || opts[:charset]
+        if conn.respond_to?(:set_client_encoding)
+          conn.set_client_encoding(encoding)
+        else
+          conn.async_exec("set client_encoding to '#{encoding}'")
+        end
+      end
+      
+      #conn.options(Mysql::OPT_LOCAL_INFILE, 'client')
+ 
+      # increase timeout so mysql server doesn't disconnect us
+      # this is especially bad if we're disconnected while EM.attach is
+      # still in progress, because by the time it gets to EM, the FD is
+      # no longer valid, and it throws a c++ 'bad file descriptor' error
+      # (do not use a timeout of -1 for unlimited, it does not work on mysqld > 5.0.60)
+      #conn.query("set @@wait_timeout = #{opts[:timeout] || 2592000}")
+
+      # we handle reconnecting (and reattaching the new fd to EM)
+      #conn.reconnect = false
+
+      # By default, MySQL 'where id is null' selects the last inserted id
+      # Turn this off. http://dev.rubyonrails.org/ticket/6778
+      #conn.query("set SQL_AUTO_IS_NULL=0")
+
+      # get results for queries
+      #conn.query_with_result = true
+
+      conn
+    rescue Exception => e
+      puts "#{e} exception"
+      if cb = opts[:errback]
+        cb.call(e)
+        nil
+      else
+        raise e
+      end
+    end
+    end
+
+    def debug(data)
+      p data if @settings[:debug]
+    end
+  end
+end

data/spec/helper.rb

@@ -0,0 +1,6 @@
+$LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
+require "rubygems"
+require "eventmachine"
+require "rspec"
+require "em-postgres"
+require "ruby-debug"

data/spec/postgres_spec.rb

@@ -0,0 +1,155 @@
+$LOAD_PATH << File.join(File.dirname(__FILE__))
+require "helper"
+require "ruby-debug"
+describe EventMachine::Postgres do
+  
+  it "should be true" do
+    true.should be_true
+  end
+  it "should create a new connection" do
+    EventMachine.run {
+      lambda {
+        conn = EventMachine::Postgres.new(:database => "test")
+        EventMachine.stop
+      }.should_not raise_error
+    }
+  end
+
+  it "should invoke errback on connection failure" do
+    EventMachine.run {
+      lambda {
+        conn = EventMachine::Postgres.new({
+            :host => 'localhost',
+            :port => 20000,
+            :socket => '',
+            :errback => Proc.new {
+              EventMachine.stop
+            }
+          })
+      }.should_not raise_error
+    }
+  end
+
+
+  it "should execute sql" do
+    EventMachine.run {
+      #EM.add_periodic_timer(1){ puts }
+      conn = EventMachine::Postgres.new(:database => "test")
+      query = conn.execute("select 1;")
+      
+      query.callback{ |res|
+        res.first["?column?"].should == "1"    
+        EventMachine.stop
+      }      
+    }
+  end
+
+  it "should accept block as query callback" do
+    EventMachine.run {
+      conn = EventMachine::Postgres.new(:database => 'test')
+      conn.execute("select 1;") { |res|
+        res.first["?column?"].should == "1"
+        EventMachine.stop
+      }
+    }
+  end
+
+
+
+  it "allow custom error callbacks for each query" do
+    EventMachine.run {
+      conn = EventMachine::Postgres.new(:database => "test")
+      query = conn.execute("select 1 from")
+      query.errback { |res|
+        #res.class.should == Mysql::Error
+        1.should == 1
+        EventMachine.stop
+        1.should == 2 #we should never get here
+      }
+    }
+  end
+
+
+  it "queue up queries and execute them in order" do
+    EventMachine.run {
+      conn = EventMachine::Postgres.new(:database => 'test')
+
+      results = []
+      #debugger
+      conn.execute("select 1 AS x;") {|res| puts res.inspect; results.push(res.first["x"].to_i)}
+      conn.execute("select 2 AS x;") {|res| puts res.inspect;results.push(res.first["x"].to_i)}
+      conn.execute("select 3 AS x;") {|res| puts res.inspect;results.push(res.first["x"].to_i)}
+      #debugger
+      EventMachine.add_timer(0.05) {
+        results.should == [1,2,3]
+        #conn.connection_pool.first.close
+        
+        EventMachine.stop
+      }
+    }
+  end
+
+
+  it "queue up large amount of queries and execute them in order" do
+    EventMachine.run {
+
+      conn = EventMachine::Postgres.new(:database => 'test')
+
+      results = []
+      (1..100).each do |i|
+        conn.execute("select #{i} AS x;") {|res| results.push(res.first["x"].to_i)}
+
+      end
+      EventMachine.add_timer(1) {
+        results.should == (1..100).to_a
+        EventMachine.stop
+      }
+    }
+  end
+
+
+  it "should continue processing queries after hitting an error" do
+    EventMachine.run {
+      conn = EventMachine::Postgres.new(:database=> 'test')
+      #errorback = Proc.new{
+      #  true.should == true
+        #EventMachine.stop
+      #}
+      q = conn.execute("select 1+ from table;") 
+      q.errback{|r| puts "hi"; true.should == true } 
+      conn.execute("select 1+1;"){ |res|
+        res.first["?column?"].to_i.should == 2
+        EventMachine.stop
+      }
+    }
+  end
+
+=begin
+  it "should work with synchronous commands" do
+    EventMachine.run {
+      conn = EventMachine::Postgres #.new(:database => 'test')
+
+      conn.list_dbs.class.should == Array
+      conn.list_tables.class.should == Array
+      conn.quote("select '1'").should == "select \\'1\\'"
+
+      EventMachine.stop
+    }
+  end
+=end
+  #  it "should reconnect when disconnected" do
+  #    # to test, run:
+  #    # mysqladmin5 -u root kill `mysqladmin -u root processlist | grep "select sleep(5)" | cut -d'|' -f2`
+  #
+  #    EventMachine.run {
+  #      conn = EventMachine::MySQL.new(:host => 'localhost')
+  #
+  #      query = conn.query("select sleep(5)")
+  #      query.callback {|res|
+  #        res.fetch_row.first.to_i.should == 0
+  #        EventMachine.stop
+  #      }
+  #    }
+  #  end
+
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification 
+name: em-postgres
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Jason Toy
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-25 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 12
+        - 9
+        version: 0.12.9
+  type: :runtime
+  version_requirements: *id001
+description: Async PostgreSQL client API for Ruby/EventMachine
+email: jtoy@jtoy.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README
+- Rakefile
+- em-postgres.gemspec
+- lib/em-postgres/postgres.rb
+- lib/em-postgres/connection.rb
+- lib/em-postgres.rb
+- spec/helper.rb
+- spec/postgres_spec.rb
+has_rdoc: true
+homepage: http://github.com/jtoy/em-postgres
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Async PostgreSQL client API for Ruby/EventMachine
+test_files: []
+