sequel 3.32.0 → 3.33.0

data/lib/sequel/adapters/shared/sqlite.rb

@@ -11,6 +11,10 @@ module Sequel
       TEMP_STORE = [:default, :file, :memory].freeze
       VIEWS_FILTER = "type = 'view'".freeze
 
+      # Whether to use integers for booleans in the database.  SQLite recommends
+      # booleans be stored as integers, but historically Sequel has used 't'/'f'.
+      attr_accessor :integer_booleans
+
       # A symbol signifying the value of the auto_vacuum PRAGMA.
       def auto_vacuum
         AUTO_VACUUM[pragma_get(:auto_vacuum).to_i]
@@ -89,6 +93,11 @@ module Sequel
       def pragma_set(name, value)
         execute_ddl("PRAGMA #{name} = #{value}")
       end
+
+      # Set the integer_booleans option using the passed in :integer_boolean option.
+      def set_integer_booleans
+        @integer_booleans = typecast_value_boolean(@opts[:integer_booleans])
+      end
       
       # The version of the server as an integer, where 3.6.19 = 30619.
       # If the server version can't be determined, 0 is used.
@@ -557,6 +566,16 @@ module Sequel
         sql << BLOB_START << v.unpack(HSTAR).first << APOS
       end
 
+      # Respect the database integer_booleans setting, using 0 or 'f'.
+      def literal_false
+        @db.integer_booleans ? '0' : "'f'"
+      end
+
+      # Respect the database integer_booleans setting, using 1 or 't'.
+      def literal_true
+        @db.integer_booleans ? '1' : "'t'"
+      end
+
       # SQLite does not support the SQL WITH clause
       def select_clause_methods
         SELECT_CLAUSE_METHODS

data/lib/sequel/adapters/sqlite.rb

@@ -1,9 +1,4 @@
 require 'sqlite3'
-begin
-  SQLite3::Database.instance_method(:type_translation)
-rescue
-  raise(Sequel::Error, "SQLite3::Database#type_translation is not defined.  If you are using the sqlite3 gem, please install the sqlite3-ruby gem.")
-end
 Sequel.require 'adapters/shared/sqlite'
 
 module Sequel
@@ -11,23 +6,69 @@ module Sequel
   # for Sequel.
   module SQLite
     TYPE_TRANSLATOR = tt = Class.new do
-      FALSE_VALUES = %w'0 false f no n'.freeze
-      def boolean(s) !FALSE_VALUES.include?(s.downcase) end
-      def integer(s) s.to_i end
-      def float(s) s.to_f end
-      def numeric(s) ::BigDecimal.new(s) rescue s end
+      FALSE_VALUES = (%w'0 false f no n' + [0]).freeze
+
+      def blob(s)
+        Sequel::SQL::Blob.new(s.to_s)
+      end
+
+      def boolean(s)
+        s = s.downcase if s.is_a?(String)
+        !FALSE_VALUES.include?(s)
+      end
+
+      def date(s)
+        case s
+        when String
+          Sequel.string_to_date(s)
+        when Integer
+          Date.jd(s)
+        when Float
+          Date.jd(s.to_i)
+        else
+          raise Sequel::Error, "unhandled type when converting to date: #{s.inspect} (#{s.class.inspect})"
+        end
+      end
+
+      def integer(s)
+        s.to_i
+      end
+
+      def float(s)
+        s.to_f
+      end
+
+      def numeric(s)
+        s = s.to_s unless s.is_a?(String)
+        ::BigDecimal.new(s) rescue s
+      end
+
+      def time(s)
+        case s
+        when String
+          Sequel.string_to_time(s)
+        when Integer
+          Sequel::SQLTime.create(s/3600, (s % 3600)/60, s % 60)
+        when Float
+          s, f = s.divmod(1)
+          Sequel::SQLTime.create(s/3600, (s % 3600)/60, s % 60, (f*1000000).round)
+        else
+          raise Sequel::Error, "unhandled type when converting to date: #{s.inspect} (#{s.class.inspect})"
+        end
+      end
+
     end.new
 
     # Hash with string keys and callable values for converting SQLite types.
     SQLITE_TYPES = {}
     {
-      %w'date' => ::Sequel.method(:string_to_date),
-      %w'time' => ::Sequel.method(:string_to_time),
+      %w'date' => tt.method(:date),
+      %w'time' => tt.method(:time),
       %w'bit bool boolean' => tt.method(:boolean),
       %w'integer smallint mediumint int bigint' => tt.method(:integer),
       %w'numeric decimal money' => tt.method(:numeric),
       %w'float double real dec fixed' + ['double precision'] => tt.method(:float),
-      %w'blob' => ::Sequel::SQL::Blob.method(:new)
+      %w'blob' => tt.method(:blob)
     }.each do |k,v|
       k.each{|n| SQLITE_TYPES[n] = v}
     end
@@ -53,8 +94,8 @@ module Sequel
       def initialize(opts={})
         super
         @conversion_procs = SQLITE_TYPES.dup
-        @conversion_procs['timestamp'] = method(:to_application_timestamp)
-        @conversion_procs['datetime'] = method(:to_application_timestamp)
+        @conversion_procs['datetime'] = @conversion_procs['timestamp'] = method(:to_application_timestamp)
+        set_integer_booleans
       end
       
       # Connect to the database.  Since SQLite is a file based database,
@@ -108,6 +149,20 @@ module Sequel
         _execute(:single_value, sql, opts)
       end
       
+      # Handle Integer and Float arguments, since SQLite can store timestamps as integers and floats.
+      def to_application_timestamp(s)
+        case s
+        when String
+          super
+        when Integer
+          super(Time.at(s).to_s)
+        when Float
+          super(DateTime.jd(s).to_s)
+        else
+          raise Sequel::Error, "unhandled type when converting to : #{s.inspect} (#{s.class.inspect})"
+        end
+      end
+
       private
       
       # Yield an available connection.  Rescue
@@ -303,7 +358,7 @@ module Sequel
             row = {}
             cols.each do |name,i,type_proc|
               v = values[i]
-              if type_proc && v.is_a?(String)
+              if type_proc && v
                 v = type_proc.call(v)
               end
               row[name] = v

data/lib/sequel/adapters/swift.rb

@@ -29,6 +29,7 @@ module Sequel
         db.extend(Sequel::Swift::SQLite::DatabaseMethods)
         db.dataset_class = Sequel::Swift::SQLite::Dataset
         db.swift_class = ::Swift::DB::Sqlite3
+        db.set_integer_booleans
       end,
     }
       

data/lib/sequel/connection_pool.rb

@@ -80,7 +80,7 @@ class Sequel::ConnectionPool
   
   # An array of symbols for all shards/servers, which is a single <tt>:default</tt> by default.
   def servers
-    [:default]
+    [DEFAULT_SERVER]
   end
   
   private

data/lib/sequel/connection_pool/sharded_single.rb

@@ -45,7 +45,7 @@ class Sequel::ShardedSingleConnectionPool < Sequel::ConnectionPool
   # This method simulates the ConnectionPool#hold API.
   def hold(server=:default)
     begin
-      server = @servers[server]
+      server = pick_server(server)
       yield(@conns[server] ||= make_new(server))
     rescue Sequel::DatabaseDisconnectError
       disconnect_server(server, &@disconnection_proc)
@@ -83,6 +83,11 @@ class Sequel::ShardedSingleConnectionPool < Sequel::ConnectionPool
       block.call(conn) if block
     end
   end
+
+  # If the server given is in the hash, return it, otherwise, return the default server.
+  def pick_server(server)
+    @servers[server]
+  end
   
   CONNECTION_POOL_MAP[[true, true]] = self
 end

data/lib/sequel/connection_pool/sharded_threaded.rb

@@ -94,7 +94,7 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # connection can be acquired, a Sequel::PoolTimeout is 
   # raised.
   def hold(server=:default)
-    sync{server = @servers[server]}
+    server = pick_server(server)
     t = Thread.current
     if conn = owned_connection(t, server)
       return yield(conn)
@@ -190,6 +190,11 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   def owned_connection(thread, server)
     sync{@allocated[server][thread]}
   end
+
+  # If the server given is in the hash, return it, otherwise, return the default server.
+  def pick_server(server)
+    sync{@servers[server]}
+  end
   
   # Releases the connection assigned to the supplied thread and server. If the
   # server or connection given is scheduled for disconnection, remove the

data/lib/sequel/connection_pool/threaded.rb

@@ -30,16 +30,19 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     @sleep_time = Float(opts[:pool_sleep_time] || 0.001)
   end
   
-  # The total number of connections opened for the given server, should
-  # be equal to available_connections.length + allocated.length.
+  # The total number of connections opened, either available or allocated.
+  # This may not be completely accurate as it isn't protected by the mutex.
   def size
     @allocated.length + @available_connections.length
   end
   
-  # Removes all connections currently available on all servers, optionally
+  # Removes all connections currently available, optionally
   # yielding each connection to the given block. This method has the effect of 
   # disconnecting from the database, assuming that no connections are currently
-  # being used.
+  # being used.  If you want to be able to disconnect connections that are
+  # currently in use, use the ShardedThreadedConnectionPool, which can do that.
+  # This connection pool does not, for performance reasons. To use the sharded pool,
+  # pass the <tt>:servers=>{}</tt> option when connecting to the database.
   # 
   # Once a connection is requested using #hold, the connection pool
   # creates new connections to the database.
@@ -51,7 +54,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     end
   end
   
-  # Chooses the first available connection to the given server, or if none are
+  # Chooses the first available connection, or if none are
   # available, creates a new connection.  Passes the connection to the supplied
   # block:
   # 
@@ -95,7 +98,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   
   private
 
-  # Assigns a connection to the supplied thread for the given server, if one
+  # Assigns a connection to the supplied thread, if one
   # is available. The calling code should NOT already have the mutex when
   # calling this.
   def acquire(thread)
@@ -106,7 +109,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     end
   end
   
-  # Returns an available connection to the given server. If no connection is
+  # Returns an available connection. If no connection is
   # available, tries to create a new connection. The calling code should already
   # have the mutex before calling this.
   def available
@@ -127,15 +130,13 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     super if (n || size) < @max_size
   end
   
-  # Returns the connection owned by the supplied thread for the given server,
+  # Returns the connection owned by the supplied thread,
   # if any. The calling code should NOT already have the mutex before calling this.
   def owned_connection(thread)
     sync{@allocated[thread]}
   end
   
-  # Releases the connection assigned to the supplied thread and server. If the
-  # server or connection given is scheduled for disconnection, remove the
-  # connection instead of releasing it back to the pool.
+  # Releases the connection assigned to the supplied thread back to the pool.
   # The calling code should already have the mutex before calling this.
   def release(thread)
     @available_connections << @allocated.delete(thread)

data/lib/sequel/database/connecting.rb

@@ -254,6 +254,8 @@ module Sequel
         else
           raise Error, 'Server opts should be a hash or proc'
         end
+      elsif server.is_a?(Hash)
+        @opts.merge(server)
       else
         @opts.dup
       end

data/lib/sequel/database/misc.rb

@@ -159,6 +159,12 @@ module Sequel
       false
     end
 
+    # Whether the database and adapter support savepoints inside prepared transactions
+    # (two-phase commit), default is false.
+    def supports_savepoints_in_prepared_transactions?
+      supports_prepared_transactions? && supports_savepoints?
+    end
+
     # Whether the database and adapter support transaction isolation levels, false by default.
     def supports_transaction_isolation_levels?
       false

data/lib/sequel/database/query.rb

@@ -39,7 +39,7 @@ module Sequel
     # Runs the supplied SQL statement string on the database server.
     # Returns self so it can be safely chained:
     #
-    #   DB << "UPADTE albums SET artist_id = NULL" << "DROP TABLE artists"
+    #   DB << "UPDATE albums SET artist_id = NULL" << "DROP TABLE artists"
     def <<(sql)
       run(sql)
       self

data/lib/sequel/extensions/arbitrary_servers.rb

@@ -0,0 +1,108 @@
+# The arbitrary_servers extension allows you to connect to arbitrary
+# servers/shards that were not defined when you created the database.
+# To use it, you first extend the Database's connection pool with the
+# Sequel::ArbitraryServers module:
+#
+#   Sequel.extension :arbitrary_servers
+#   DB.pool.extend Sequel::ArbitraryServers
+#
+# Then you can pass arbitrary connection options for the server/shard
+# to use as a hash:
+#
+#   DB[:table].server(:host=>'...', :database=>'...').all
+#
+# Because Sequel can never be sure that the connection will be reused,
+# arbitrary connections are disconnected as soon as the outermost block
+# that uses them exits.  So this example uses the same connection:
+#
+#   DB.transaction(:server=>{:host=>'...', :database=>'...'}) do |c|
+#     DB.transaction(:server=>{:host=>'...', :database=>'...'}) do |c2|
+#       # c == c2
+#     end
+#   end
+#
+# But this example does not:
+#
+#   DB.transaction(:server=>{:host=>'...', :database=>'...'}) do |c|
+#   end
+#   DB.transaction(:server=>{:host=>'...', :database=>'...'}) do |c2|
+#     # c != c2
+#   end
+#
+# You can use this extension in conjunction with the server_block
+# extension:
+#
+#   DB.with_server(:host=>'...', :database=>'...') do
+#     DB.synchronize do
+#       # All of these use the host/database given to with_server
+#       DB[:table].insert(...)
+#       DB[:table].update(...)
+#       DB.tables
+#       DB[:table].all
+#     end
+#   end
+#
+# Anyone using this extension in conjunction with the server_block
+# extension may want to do the following to so that you don't need
+# to call synchronize separately:
+#
+#   def DB.with_server(*)
+#     super{synchronize{yield}}
+#   end
+#
+# Note that this extension only works with the sharded threaded connection
+# pool.  If you are using the sharded single connection pool, you need
+# to switch to the sharded threaded connection pool before using this
+# extension.
+
+module Sequel
+  module ArbitraryServers
+    private
+
+    # If server is a hash, create a new connection for
+    # it, and cache it first by thread and then server.
+    def acquire(thread, server)
+      if server.is_a?(Hash)
+        sync{@allocated[thread] ||= {}}[server] = make_new(server)
+      else
+        super
+      end
+    end
+    
+    # If server is a hash, the entry for it probably doesn't
+    # exist in the @allocated hash, so check for existence to
+    # avoid calling nil.[]
+    def owned_connection(thread, server)
+      if server.is_a?(Hash)
+        if a = sync{@allocated[thread]}
+          a[server]
+        end
+      else
+        super
+      end
+    end
+
+    # If server is a hash, return it directly.
+    def pick_server(server)
+      if server.is_a?(Hash)
+        server
+      else
+        super
+      end
+    end
+
+    # If server is a hash, delete the thread from the allocated
+    # connections for that server.  Additionally, if this was the last thread
+    # using that server, delete the server from the @allocated hash.
+    def release(thread, conn, server)
+      if server.is_a?(Hash)
+        a = @allocated[thread]
+        a.delete(server)
+        @allocated.delete(thread) if a.empty?
+        @disconnection_proc.call(conn) if @disconnection_proc
+      else  
+        super
+      end
+    end
+  end
+end

data/lib/sequel/extensions/migration.rb

@@ -22,7 +22,7 @@ module Sequel
   #
   # Part of the +migration+ extension.
   class Migration
-    # Creates a new instance of the migration and sets the @db attribute.
+    # Set the database associated with this migration.
     def initialize(db)
       @db = db
     end
@@ -43,6 +43,11 @@ module Sequel
     def self.inherited(base)
       descendants << base
     end
+
+    # Always use transactions for old-style migrations.
+    def self.use_transactions
+      true
+    end
     
     # The default down action does nothing
     def down
@@ -69,6 +74,14 @@ module Sequel
     # Proc used for the up action
     attr_accessor :up
 
+    # Whether to use transactions for this migration, true by default.
+    attr_accessor :use_transactions
+
+    # Use transactions by default
+    def initialize
+      @use_transactions = true
+    end
+
     # Apply the appropriate block on the +Database+
     # instance using instance_eval.
     def apply(db, direction)
@@ -100,6 +113,11 @@ module Sequel
       migration.down = block
     end
 
+    # Disable the use of transactions for the related migration
+    def no_transaction
+      migration.use_transactions = false
+    end
+
     # Defines the migration's up action.
     def up(&block)
       migration.up = block
@@ -337,11 +355,15 @@ module Sequel
     # if the version number appears to be a unix time integer for a year
     # after 2005, otherwise uses the IntegerMigrator.
     def self.migrator_class(directory)
-      Dir.new(directory).each do |file|
-        next unless MIGRATION_FILE_PATTERN.match(file)
-        return TimestampMigrator if file.split(MIGRATION_SPLITTER, 2).first.to_i > MINIMUM_TIMESTAMP
+      if self.equal?(Migrator)
+        Dir.new(directory).each do |file|
+          next unless MIGRATION_FILE_PATTERN.match(file)
+          return TimestampMigrator if file.split(MIGRATION_SPLITTER, 2).first.to_i > MINIMUM_TIMESTAMP
+        end
+        IntegerMigrator
+      else
+        self
       end
-      IntegerMigrator
     end
     private_class_method :migrator_class
     
@@ -380,10 +402,23 @@ module Sequel
       @table = schema ? Sequel::SQL::QualifiedIdentifier.new(schema, table) : table
       @column = opts[:column] || self.class.const_get(:DEFAULT_SCHEMA_COLUMN)
       @ds = schema_dataset
+      @use_transactions = opts[:use_transactions]
     end
 
     private
 
+    # If transactions should be used for the migration, yield to the block
+    # inside a transaction.  Otherwise, just yield to the block.
+    def checked_transaction(migration, &block)
+      if @use_transactions == false
+        yield
+      elsif migration.use_transactions || @use_transactions
+        db.transaction(&block)
+      else
+        yield
+      end
+    end
+
     # Remove all migration classes.  Done by the migrator to ensure that
     # the correct migration classes are picked up.
     def remove_migration_classes
@@ -437,7 +472,7 @@ module Sequel
         t = Time.now
         lv = up? ? v : v + 1
         db.log_info("Begin applying migration version #{lv}, direction: #{direction}")
-        db.transaction do
+        checked_transaction(m) do
           m.apply(db, direction)
           set_migration_version(v)
         end
@@ -461,6 +496,9 @@ module Sequel
       Dir.new(directory).each do |file|
         next unless MIGRATION_FILE_PATTERN.match(file)
         version = migration_version_from_file(file)
+        if version >= 20000101
+          raise Migrator::Error, "Migration number too large, must use TimestampMigrator: #{file}"
+        end
         raise(Error, "Duplicate migration version: #{version}") if files[version]
         files[version] = File.join(directory, file)
       end
@@ -550,7 +588,7 @@ module Sequel
       migration_tuples.each do |m, f, direction|
         t = Time.now
         db.log_info("Begin applying migration #{f}, direction: #{direction}")
-        db.transaction do
+        checked_transaction(m) do
           m.apply(db, direction)
           fi = f.downcase
           direction == :up ? ds.insert(column=>fi) : ds.filter(column=>fi).delete