sequel 3.7.0 → 3.8.0

data/lib/sequel/adapters/shared/mssql.rb

@@ -8,7 +8,7 @@ module Sequel
       SQL_BEGIN = "BEGIN TRANSACTION".freeze
       SQL_COMMIT = "COMMIT TRANSACTION".freeze
       SQL_ROLLBACK = "IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION".freeze
-      SQL_ROLLBACK_TO_SAVEPOINT = 'ROLLBACK TRANSACTION autopoint_%d'.freeze
+      SQL_ROLLBACK_TO_SAVEPOINT = 'IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION autopoint_%d'.freeze
       SQL_SAVEPOINT = 'SAVE TRANSACTION autopoint_%d'.freeze
       TEMPORARY = "#".freeze
       
@@ -183,7 +183,7 @@ module Sequel
       COMMA_SEPARATOR = ', '.freeze
       DELETE_CLAUSE_METHODS = Dataset.clause_methods(:delete, %w'with from output from2 where')
       INSERT_CLAUSE_METHODS = Dataset.clause_methods(:insert, %w'with into columns output values')
-      SELECT_CLAUSE_METHODS = Dataset.clause_methods(:select, %w'with limit distinct columns from table_options join where group order having compounds')
+      SELECT_CLAUSE_METHODS = Dataset.clause_methods(:select, %w'with limit distinct columns from table_options join where group having order compounds')
       UPDATE_CLAUSE_METHODS = Dataset.clause_methods(:update, %w'with table set output from where')
       WILDCARD = LiteralString.new('*').freeze
       CONSTANT_MAP = {:CURRENT_DATE=>'CAST(CURRENT_TIMESTAMP AS DATE)'.freeze, :CURRENT_TIME=>'CAST(CURRENT_TIMESTAMP AS TIME)'.freeze}
@@ -230,6 +230,7 @@ module Sequel
 
       # Use the OUTPUT clause to get the value of all columns for the newly inserted record.
       def insert_select(*values)
+        return unless supports_output_clause?
         naked.clone(default_server_opts(:sql=>output(nil, [:inserted.*]).insert_sql(*values))).single_record unless opts[:disable_insert_output]
       end
 
@@ -256,6 +257,7 @@ module Sequel
       #   dataset.output(:output_table, [:deleted__id, :deleted__name])
       #   dataset.output(:output_table, :id => :inserted__id, :name => :inserted__name)
       def output(into, values)
+        raise(Error, "SQL Server versions 2000 and earlier do not support the OUTPUT clause") unless supports_output_clause?
         output = {}
         case values
           when Hash
@@ -332,6 +334,11 @@ module Sequel
       def supports_multiple_column_in?
         false
       end
+      
+      # Only 2005+ supports the output clause.
+      def supports_output_clause?
+        server_version >= 9000000
+      end
 
       # MSSQL 2005+ supports window functions
       def supports_window_functions?
@@ -388,9 +395,9 @@ module Sequel
         blob
       end
       
-      # Use unicode string syntax for all strings
+      # Use unicode string syntax for all strings. Don't double backslashes.
       def literal_string(v)
-        "N#{super}"
+        "N'#{v.gsub(/'/, "''")}'"
       end
       
       # Use 0 for false on MSSQL
@@ -425,6 +432,7 @@ module Sequel
 
       # SQL fragment for MSSQL's OUTPUT clause.
       def output_sql(sql)
+        return unless supports_output_clause?
         return unless output = @opts[:output]
         sql << " OUTPUT #{column_list(output[:select_list])}"
         if into = output[:into]

data/lib/sequel/adapters/shared/mysql.rb

@@ -179,6 +179,22 @@ module Sequel
         end
       end
 
+      # Respect the :size option if given to produce
+      # tinyblob, mediumblob, and longblob if :tiny,
+      # :medium, or :long is given.
+      def type_literal_generic_file(column)
+        case column[:size]
+        when :tiny    # < 2^8 bytes
+          :tinyblob
+        when :medium  # < 2^24 bytes
+          :mediumblob
+        when :long    # < 2^32 bytes
+          :longblob
+        else          # 2^16 bytes
+          :blob
+        end
+      end
+
       # MySQL has both datetime and timestamp classes, most people are going
       # to want datetime
       def type_literal_generic_datetime(column)

data/lib/sequel/connection_pool.rb

@@ -45,39 +45,75 @@ class Sequel::ConnectionPool
     raise(Sequel::Error, ':max_connections must be positive') if @max_size < 1
     @mutex = Mutex.new
     @connection_proc = block
-    @disconnection_proc = opts[:disconnection_proc]
-    @servers = [:default]
-    @servers += opts[:servers].keys - @servers if opts[:servers] 
-    @available_connections = Hash.new{|h,k| h[:default]}
-    @allocated = Hash.new{|h,k| h[:default]}
-    @servers.each do |s|
-      @available_connections[s] = []
-      @allocated[s] = {}
-    end
+    @disconnection_proc = opts[:disconnection_proc]    
+    @available_connections = {}
+    @allocated = {}
+    @connections_to_remove = []
+    @servers = Hash.new(:default)
+    add_servers([:default])
+    add_servers(opts[:servers].keys) if opts[:servers]
     @timeout = Integer(opts[:pool_timeout] || 5)
     @sleep_time = Float(opts[:pool_sleep_time] || 0.001)
     @convert_exceptions = opts.include?(:pool_convert_exceptions) ? opts[:pool_convert_exceptions] : true
   end
   
+  # Adds new servers to the connection pool. Primarily used in conjunction with master/slave
+  # or shard configurations. Allows for dynamic expansion of the potential slaves/shards
+  # at runtime. servers argument should be an array of symbols. 
+  def add_servers(servers)
+    sync do
+      servers.each do |server|
+        unless @servers.has_key?(server)
+          @servers[server] = server
+          @available_connections[server] = []
+          @allocated[server] = {}
+        end
+      end
+    end
+  end
+  
   # A hash of connections currently being used for the given server, key is the
-  # Thread, value is the connection.
+  # Thread, value is the connection.  Nonexistent servers will return nil.  Treat
+  # this as read only, do not modify the resulting object.
   def allocated(server=:default)
     @allocated[server]
   end
   
   # An array of connections opened but not currently used, for the given
-  # server.
+  # server. Nonexistent servers will return nil. Treat this as read only, do
+  # not modify the resulting object.
   def available_connections(server=:default)
     @available_connections[server]
   end
   
   # The total number of connections opened for the given server, should
-  # be equal to available_connections.length + allocated.length
+  # be equal to available_connections.length + allocated.length.  Nonexistent
+  # servers will return the created count of the default server.
   def created_count(server=:default)
+    server = @servers[server]
     @allocated[server].length + @available_connections[server].length
   end
   alias size created_count
   
+  # Removes all connection currently available on all servers, optionally
+  # yielding each connection to the given block. This method has the effect of 
+  # disconnecting from the database, assuming that no connections are currently
+  # being used.  If connections are being used, they are scheduled to be
+  # disconnected as soon as they are returned to the pool.
+  # 
+  # Once a connection is requested using #hold, the connection pool
+  # creates new connections to the database. Options:
+  # * :server - Should be a symbol specifing the server to disconnect from,
+  #   or an array of symbols to specify multiple servers.
+  def disconnect(opts={}, &block)
+    block ||= @disconnection_proc
+    sync do
+      (opts[:server] ? Array(opts[:server]) : @servers.keys).each do |s|
+        disconnect_server(s, &block)
+      end
+    end
+  end
+  
   # Chooses the first available connection to the given server, or if none are
   # available, creates a new connection.  Passes the connection to the supplied
   # block:
@@ -94,27 +130,28 @@ class Sequel::ConnectionPool
   # raised.
   def hold(server=:default)
     begin
+      sync{server = @servers[server]}
       t = Thread.current
       if conn = owned_connection(t, server)
         return yield(conn)
       end
       begin
         unless conn = acquire(t, server)
-          time = Time.new
+          time = Time.now
           timeout = time + @timeout
           sleep_time = @sleep_time
           sleep sleep_time
           until conn = acquire(t, server)
-            raise(::Sequel::PoolTimeout) if Time.new > timeout
+            raise(::Sequel::PoolTimeout) if Time.now > timeout
             sleep sleep_time
           end
         end
         yield conn
-      rescue Sequel::DatabaseDisconnectError => dde
-        remove(t, conn, server) if conn
+      rescue Sequel::DatabaseDisconnectError
+        sync{@connections_to_remove << conn} if conn
         raise
       ensure
-        @mutex.synchronize{release(t, server)} if conn && !dde
+        sync{release(t, conn, server)} if conn
       end
     rescue StandardError 
       raise
@@ -122,28 +159,37 @@ class Sequel::ConnectionPool
       raise(@convert_exceptions ? RuntimeError.new(e.message) : e)
     end
   end
-  
-  # Removes all connection currently available on all servers, optionally
-  # yielding each connection to the given block. This method has the effect of 
-  # disconnecting from the database, assuming that no connections are currently
-  # being used. Once a connection is requested using #hold, the connection pool
-  # creates new connections to the database.
-  def disconnect(&block)
-    block ||= @disconnection_proc
-    @mutex.synchronize do
-      @available_connections.each do |server, conns|
-        conns.each{|c| block.call(c)} if block
-        conns.clear
+
+  # Remove servers from the connection pool. Primarily used in conjunction with master/slave
+  # or shard configurations.  Similar to disconnecting from all given servers,
+  # except that after it is used, future requests for the server will use the
+  # :default server instead.
+  def remove_servers(servers)
+    sync do
+      raise(Sequel::Error, "cannot remove default server") if servers.include?(:default)
+      servers.each do |server|
+        if @servers.include?(server)
+          disconnect_server(server, &@disconnection_proc)
+          @available_connections.delete(server)
+          @allocated.delete(server)
+          @servers.delete(server)
+        end
       end
     end
   end
-  
+
+  # Return an array of symbols for servers in the connection pool.
+  def servers
+    sync{@servers.keys}
+  end
+
   private
 
   # Assigns a connection to the supplied thread for the given server, if one
-  # is available.
+  # is available. The calling code should NOT already have the mutex when
+  # calling this.
   def acquire(thread, server)
-    @mutex.synchronize do
+    sync do
       if conn = available(server)
         allocated(server)[thread] = conn
       end
@@ -151,16 +197,30 @@ class Sequel::ConnectionPool
   end
   
   # Returns an available connection to the given server. If no connection is
-  # available, tries to create a new connection.
+  # available, tries to create a new connection. The calling code should already
+  # have the mutex before calling this.
   def available(server)
     available_connections(server).pop || make_new(server)
   end
-  
+
+  # Disconnect from the given server.  Disconnects available connections
+  # immediately, and schedules currently allocated connections for disconnection
+  # as soon as they are returned to the pool. The calling code should already
+  # have the mutex before calling this.
+  def disconnect_server(server, &block)
+    if conns = available_connections(server)
+      conns.each{|conn| block.call(conn)} if block
+      conns.clear
+    end
+    @connections_to_remove.concat(allocated(server).values)
+  end
+
   # Creates a new connection to the given server if the size of the pool for
-  # the server is less than the maximum size of the pool.
+  # the server is less than the maximum size of the pool. The calling code
+  # should already have the mutex before calling this.
   def make_new(server)
     if (n = created_count(server)) >= @max_size
-      allocated(server).keys.reject{|t| t.alive?}.each{|t| release(t, server)}
+      allocated(server).to_a.each{|t, c| release(t, c, server) unless t.alive?}
       n = nil
     end
     if (n || created_count(server)) < @max_size
@@ -176,23 +236,35 @@ class Sequel::ConnectionPool
   end
   
   # Returns the connection owned by the supplied thread for the given server,
-  # if any.
+  # if any. The calling code should NOT already have the mutex before calling this.
   def owned_connection(thread, server)
-    @mutex.synchronize{@allocated[server][thread]}
+    sync{@allocated[server][thread]}
   end
   
-  # Releases the connection assigned to the supplied thread and server.
-  # You must already have the mutex before you call this.
-  def release(thread, server)
-    available_connections(server) << allocated(server).delete(thread)
+  # Releases the connection assigned to the supplied thread and server. If the
+  # server or connection given is scheduled for disconnection, remove the
+  # connection instead of releasing it back to the pool.
+  # The calling code should already have the mutex before calling this.
+  def release(thread, conn, server)
+    if @connections_to_remove.include?(conn)
+      remove(thread, conn, server)
+    else
+      available_connections(server) << allocated(server).delete(thread)
+    end
   end
 
-  # Removes the currently allocated connection from the connection pool.
+  # Removes the currently allocated connection from the connection pool. The
+  # calling code should already have the mutex before calling this.
   def remove(thread, conn, server)
-    @mutex.synchronize do
-      allocated(server).delete(thread)
-      @disconnection_proc.call(conn) if @disconnection_proc
-    end
+    @connections_to_remove.delete(conn)
+    allocated(server).delete(thread) if @servers.include?(server)
+    @disconnection_proc.call(conn) if @disconnection_proc
+  end
+  
+  # Yield to the block while inside the mutex. The calling code should NOT
+  # already have the mutex before calling this.
+  def sync
+    @mutex.synchronize{yield}
   end
 end
 
@@ -220,12 +292,31 @@ class Sequel::SingleThreadedPool
     @connection_proc = block
     @disconnection_proc = opts[:disconnection_proc]
     @conns = {}
+    @servers = Hash.new(:default)
+    add_servers([:default])
+    add_servers(opts[:servers].keys) if opts[:servers]
     @convert_exceptions = opts.include?(:pool_convert_exceptions) ? opts[:pool_convert_exceptions] : true
   end
   
+  # Adds new servers to the connection pool. Primarily used in conjunction with master/slave
+  # or shard configurations. Allows for dynamic expansion of the potential slaves/shards
+  # at runtime. servers argument should be an array of symbols. 
+  def add_servers(servers)
+    servers.each{|s| @servers[s] = s}
+  end
+  
   # The connection for the given server.
   def conn(server=:default)
-    @conns[server]
+    @conns[@servers[server]]
+  end
+  
+  # Disconnects from the database. Once a connection is requested using
+  # #hold, the connection is reestablished. Options:
+  # * :server - Should be a symbol specifing the server to disconnect from,
+  #   or an array of symbols to specify multiple servers.
+  def disconnect(opts={}, &block)
+    block ||= @disconnection_proc
+    (opts[:server] ? Array(opts[:server]) : servers).each{|s| disconnect_server(s, &block)}
   end
   
   # Yields the connection to the supplied block for the given server.
@@ -233,10 +324,10 @@ class Sequel::SingleThreadedPool
   def hold(server=:default)
     begin
       begin
-        yield(c = (@conns[server] ||= @connection_proc.call(server)))
+        server = @servers[server]
+        yield(c = (@conns[server] ||= make_new(server)))
       rescue Sequel::DatabaseDisconnectError
-        @conns.delete(server)
-        @disconnection_proc.call(c) if @disconnection_proc
+        disconnect_server(server, &@disconnection_proc)
         raise
       end
     rescue Exception => e
@@ -245,11 +336,41 @@ class Sequel::SingleThreadedPool
     end
   end
   
-  # Disconnects from the database. Once a connection is requested using
-  # #hold, the connection is reestablished.
-  def disconnect(&block)
-    block ||= @disconnection_proc
-    @conns.values.each{|conn| block.call(conn) if block}
-    @conns = {}
+  # Remove servers from the connection pool. Primarily used in conjunction with master/slave
+  # or shard configurations.  Similar to disconnecting from all given servers,
+  # except that after it is used, future requests for the server will use the
+  # :default server instead.
+  def remove_servers(servers)
+    raise(Sequel::Error, "cannot remove default server") if servers.include?(:default)
+    servers.each do |server|
+      disconnect_server(server, &@disconnection_proc)
+      @servers.delete(server)
+    end
+  end
+  
+  # Return an array of symbols for servers in the connection pool.
+  def servers
+    @servers.keys
+  end
+  
+  private
+  
+  # Disconnect from the given server, if connected.
+  def disconnect_server(server, &block)
+    if conn = @conns.delete(server)
+      block.call(conn) if block
+    end
+  end
+  
+  # Return a connection to the given server, raising DatabaseConnectionError
+  # if the connection_proc raises an error or doesn't return a valid connection.
+  def make_new(server)
+    begin
+      conn = @connection_proc.call(server)
+    rescue Exception=>exception
+      raise Sequel.convert_exception_class(exception, Sequel::DatabaseConnectionError)
+    end
+    raise(Sequel::DatabaseConnectionError, "Connection parameters not valid") unless conn
+    conn
   end
 end

data/lib/sequel/database.rb

@@ -255,6 +255,20 @@ module Sequel
       (String === args.first) ? fetch(*args) : from(*args)
     end
     
+    # Dynamically add new servers or modify server options at runtime. Also adds new
+    # servers to the connection pool. Intended for use with master/slave or shard
+    # configurations where it is useful to add new server hosts at runtime.
+    #
+    # servers argument should be a hash with server name symbol keys and hash or
+    # proc values.  If a servers key is already in use, it's value is overridden
+    # with the value provided.
+    #
+    #  DB.add_servers(:f=>{:host=>"hash_host_f"})
+    def add_servers(servers)
+      @opts[:servers] = @opts[:servers] ? @opts[:servers].merge(servers) : servers
+      @pool.add_servers(servers.keys)
+    end
+    
     # Call the prepared statement with the given name with the given hash
     # of arguments.
     def call(ps_name, hash={})
@@ -288,9 +302,20 @@ module Sequel
     end
     
     # Disconnects all available connections from the connection pool.  Any
-    # connections currently in use will not be disconnected.
-    def disconnect
-      pool.disconnect
+    # connections currently in use will not be disconnected. Options:
+    # * :servers - Should be a symbol specifing the server to disconnect from,
+    #   or an array of symbols to specify multiple servers.
+    def disconnect(opts = {})
+      pool.disconnect(opts)
+    end
+
+    # Yield a new database object for every server in the connection pool.
+    # Intended for use in sharded environments where there is a need to make schema
+    # modifications (DDL queries) on each shard.
+    #
+    #   DB.each_server{|db| db.create_table(:users){primary_key :id; String :name}}
+    def each_server(&block)
+      servers.each{|s| self.class.connect(server_opts(s), &block)}
     end
 
     # Executes the given SQL on the database. This method should be overridden in descendants.
@@ -431,6 +456,26 @@ module Sequel
       @quote_identifiers = @opts.include?(:quote_identifiers) ? @opts[:quote_identifiers] : (@@quote_identifiers.nil? ? quote_identifiers_default : @@quote_identifiers)
     end
     
+    # Dynamically remove existing servers from the connection pool. Intended for
+    # use with master/slave or shard configurations where it is useful to remove
+    # existing server hosts at runtime.
+    #
+    # servers should be symbols or arrays of symbols.  If a nonexistent server
+    # is specified, it is ignored.  If no servers have been specified for
+    # this database, no changes are made. If you attempt to remove the :default server,
+    # an error will be raised.
+    #
+    #   DB.remove_servers(:f1, :f2)
+    def remove_servers(*servers)
+      if @opts[:servers] && !@opts[:servers].empty?
+        servs = @opts[:servers].dup
+        servers.flatten!
+        servers.each{|s| servs.delete(s)}
+        @opts[:servers] = servs
+        @pool.remove_servers(servers)
+      end
+    end
+    
     # Runs the supplied SQL statement string on the database server. Returns nil.
     # Options:
     # * :server - The server to run the SQL on.
@@ -439,11 +484,6 @@ module Sequel
       nil
     end
     
-    # Returns a new dataset with the select method invoked.
-    def select(*args, &block)
-      dataset.select(*args, &block)
-    end
-    
     # Parse the schema from the database.
     # Returns the schema for the given table as an array with all members being arrays of length 2,
     # the first member being the column name, and the second member being a hash of column information.
@@ -472,6 +512,16 @@ module Sequel
       @schemas[quoted_name] = cols
     end
 
+    # Returns a new dataset with the select method invoked.
+    def select(*args, &block)
+      dataset.select(*args, &block)
+    end
+
+    # An array of servers/shards for this Database object.
+    def servers
+      pool.servers
+    end
+
     # Returns true if the database is using a single-threaded connection pool.
     def single_threaded?
       @single_threaded
@@ -488,8 +538,7 @@ module Sequel
     end
 
     # Returns true if a table with the given name exists.  This requires a query
-    # to the database unless this database object already has the schema for
-    # the given table name.
+    # to the database.
     def table_exists?(name)
       begin 
         from(name).first
@@ -905,7 +954,7 @@ module Sequel
       opts.delete(:servers)
       opts
     end
-
+    
     # Raise a database error unless the exception is an Rollback.
     def transaction_error(e)
       raise_error(e, :classes=>database_error_classes) unless e.is_a?(Rollback)
@@ -996,4 +1045,3 @@ module Sequel
     end
   end
 end
-