sequel 3.5.0 → 3.6.0

data/doc/schema.rdoc

@@ -11,21 +11,26 @@ Migrations are not required, you can just call the schema modification methods d
 Sequel has the ability to have database independent migrations using ruby classes as types.  When you use a ruby class as a type, Sequel translates it to the most comparable type in the database you are using.  Here's an example using all supported types:
 
     DB.create_table(:cats) do
-      primary_key :id, :type=>Integer # integer
-      String :a                       # varchar(255)
-      column :b, File                 # blob
-      Fixnum :c                       # integer
+      primary_key :id, :type=>Integer             # integer
+      String :a                                   # varchar(255)
+      String :a2, :size=>50                       # varchar(50)
+      String :a3, :fixed=>true                    # char(255)
+      String :a4, :fixed=>true, :size=>50         # char(50)
+      String :a5, :text=>true                     # text
+      column :b, File                             # blob
+      Fixnum :c                                   # integer
       foreign_key :d, :other_table, :type=>Bignum # bigint
-      Float :e                        # double precision
-      BigDecimal :f                   # numeric
-      Date :g                         # date
-      DateTime :h                     # timestamp
-      Time :i                         # timestamp
-      Numeric :j                      # numeric
-      TrueClass :k                    # boolean
-      FalseClass :l                   # boolean
+      Float :e                                    # double precision
+      BigDecimal :f                               # numeric
+      BigDecimal :f2, :size=>10                   # numeric(10)
+      BigDecimal :f3, :size=>[10, 2]              # numeric(10, 2)
+      Date :g                                     # date
+      DateTime :h                                 # timestamp
+      Time :i                                     # timestamp
+      Time :i2, :only_time=>true                  # time
+      Numeric :j                                  # numeric
+      TrueClass :k                                # boolean
+      FalseClass :l                               # boolean
     end
 
 Basically, if you use one of the ruby classes above, it will translate into a database specific type.  If you use a lowercase method, symbol, or string to specify the type, Sequel won't attempt to translate it.
-
-Note that if you use a ruby class, you shouldn't use a :size option.  Using a ruby class means that you want Sequel to pick the database type to use.  If you want to specify the size, you are no longer in database independent territory, and you need to specify the type as well, using a lowercase method, symbol, or string.

data/lib/sequel/adapters/amalgalite.rb

@@ -82,9 +82,9 @@ module Sequel
         Amalgalite::Dataset.new(self, opts)
       end
       
-      # Run the given SQL with the given arguments and reload the schema.  Returns nil.
+      # Run the given SQL with the given arguments. Returns nil.
       def execute_ddl(sql, opts={})
-        _execute(sql, opts){|conn| conn.execute_batch(sql); conn.reload_schema!}
+        _execute(sql, opts){|conn| conn.execute_batch(sql);}
         nil
       end
       
@@ -99,19 +99,8 @@ module Sequel
       end
       
       # Run the given SQL with the given arguments and yield each row.
-      def execute(sql, opts={})
-        retried = false
-        _execute(sql, opts) do |conn|
-          conn.prepare(sql) do |stmt|
-            begin
-             stmt.result_meta
-            rescue NoMethodError
-              conn.reload_schema!
-              stmt.result_meta
-            end
-            yield stmt
-          end
-        end
+      def execute(sql, opts={}, &block)
+        _execute(sql, opts){|conn| conn.prepare(sql, &block)}
       end
       
       # Run the given SQL with the given arguments and return the first value of the first row.
@@ -158,20 +147,9 @@ module Sequel
     class Dataset < Sequel::Dataset
       include ::Sequel::SQLite::DatasetMethods
       
-      EXPLAIN = 'EXPLAIN %s'.freeze
-      
-      # Return an array of strings specifying a query explanation for the
-      # current dataset.
-      def explain
-        res = []
-        @db.result_set(EXPLAIN % select_sql(opts), nil) {|r| res << r}
-        res
-      end
-      
       # Yield a hash for each row in the dataset.
       def fetch_rows(sql)
         execute(sql) do |stmt|
-          stmt.result_meta
           @columns = cols = stmt.result_fields.map{|c| output_identifier(c)}
           col_count = cols.size
           stmt.each do |result|
@@ -186,7 +164,7 @@ module Sequel
       
       # Quote the string using the adapter instance method.
       def literal_string(v)
-        "#{db.synchronize{|c| c.quote(v)}}"
+        db.synchronize{|c| c.quote(v)}
       end
     end
   end

data/lib/sequel/adapters/jdbc.rb

@@ -167,7 +167,7 @@ module Sequel
                 stmt.execute(sql)
               when :insert
                 stmt.executeUpdate(sql)
-                last_insert_id(conn, opts)
+                last_insert_id(conn, opts.merge(:stmt=>stmt))
               else
                 stmt.executeUpdate(sql)
               end
@@ -303,6 +303,14 @@ module Sequel
         end
       end
       
+      # Support fractional seconds for Time objects used in bound variables
+      def java_sql_timestamp(time)
+        millis = time.to_i * 1000
+        ts = java.sql.Timestamp.new(millis)
+        ts.setNanos(time.usec * 1000)
+        ts
+      end
+
       # By default, there is no support for determining the last inserted
       # id, so return nil.  This method should be overridden in
       # sub adapters.
@@ -335,8 +343,10 @@ module Sequel
           cps.setString(i, arg)
         when Date, Java::JavaSql::Date
           cps.setDate(i, arg)
-        when Time, DateTime, Java::JavaSql::Timestamp
+        when DateTime, Java::JavaSql::Timestamp
           cps.setTimestamp(i, arg)
+        when Time
+          cps.setTimestamp(i, java_sql_timestamp(arg))
         when Float
           cps.setDouble(i, arg)
         when TrueClass, FalseClass
@@ -455,7 +465,7 @@ module Sequel
       
       # Create a named prepared statement that is stored in the
       # database (and connection) for reuse.
-      def prepare(type, name=nil, values=nil)
+      def prepare(type, name=nil, *values)
         ps = to_prepared_statement(type, values)
         ps.extend(PreparedStatementMethods)
         if name

data/lib/sequel/adapters/jdbc/h2.rb

@@ -76,6 +76,18 @@ module Sequel
       class Dataset < JDBC::Dataset
         SELECT_CLAUSE_METHODS = clause_methods(:select, %w'distinct columns from join where group having compounds order limit')
         
+        # Work around H2's lack of a case insensitive LIKE operator
+        def complex_expression_sql(op, args)
+          case op
+          when :ILIKE
+            super(:LIKE, [SQL::PlaceholderLiteralString.new("CAST(? AS VARCHAR_IGNORECASE)", [args.at(0)]), args.at(1)])
+          when :"NOT ILIKE"
+            super(:"NOT LIKE", [SQL::PlaceholderLiteralString.new("CAST(? AS VARCHAR_IGNORECASE)", [args.at(0)]), args.at(1)])
+          else
+            super(op, args)
+          end
+        end
+        
         # H2 requires SQL standard datetimes
         def requires_sql_standard_datetimes?
           true
@@ -86,6 +98,11 @@ module Sequel
           false
         end
         
+        # H2 doesn't support JOIN USING
+        def supports_join_using?
+          false
+        end 
+
         private
       
         def select_clause_methods

data/lib/sequel/adapters/jdbc/mysql.rb

@@ -26,13 +26,26 @@ module Sequel
         
         # Get the last inserted id using LAST_INSERT_ID().
         def last_insert_id(conn, opts={})
-          stmt = conn.createStatement
-          begin
-            rs = stmt.executeQuery('SELECT LAST_INSERT_ID()')
-            rs.next
-            rs.getInt(1)
-          ensure
-            stmt.close
+          if stmt = opts[:stmt]
+            rs = stmt.getGeneratedKeys
+            begin
+              if rs.next
+                rs.getInt(1)
+              else
+                0
+              end
+            ensure
+              rs.close
+            end
+          else
+            stmt = conn.createStatement
+            begin
+              rs = stmt.executeQuery('SELECT LAST_INSERT_ID()')
+              rs.next
+              rs.getInt(1)
+            ensure
+              stmt.close
+            end
           end
         end
       end

data/lib/sequel/adapters/mysql.rb

@@ -257,6 +257,7 @@ module Sequel
         def subselect_sql(ds)
           ps = ds.to_prepared_statement(:select)
           ps.extend(CallableStatementMethods)
+          ps = ps.bind(@opts[:bind_vars]) if @opts[:bind_vars]
           ps.prepared_args = prepared_args
           ps.prepared_sql
         end
@@ -302,10 +303,10 @@ module Sequel
       # breaks the use of subselects in prepared statements, so extend the
       # temporary prepared statement that this creates with a module that
       # fixes it.
-      def call(type, bind_arguments={}, values=nil)
+      def call(type, bind_arguments={}, *values, &block)
         ps = to_prepared_statement(type, values)
         ps.extend(CallableStatementMethods)
-        ps.call(bind_arguments)
+        ps.call(bind_arguments, &block)
       end
       
       # Delete rows matching this dataset
@@ -345,7 +346,7 @@ module Sequel
       
       # Store the given type of prepared statement in the associated database
       # with the given name.
-      def prepare(type, name=nil, values=nil)
+      def prepare(type, name=nil, *values)
         ps = to_prepared_statement(type, values)
         ps.extend(PreparedStatementMethods)
         if name

data/lib/sequel/adapters/oracle.rb

@@ -91,7 +91,7 @@ module Sequel
       end
       
       def remove_transaction(conn)
-        conn.autocommit = true
+        conn.autocommit = true if conn
         super
       end
       

data/lib/sequel/adapters/postgres.rb

@@ -84,6 +84,7 @@ rescue LoadError => e
 end
 
 module Sequel
+  Dataset::NON_SQL_OPTIONS << :cursor
   module Postgres
     CONVERTED_EXCEPTIONS << PGError
     
@@ -304,22 +305,28 @@ module Sequel
       
       # Yield all rows returned by executing the given SQL and converting
       # the types.
-      def fetch_rows(sql)
-        cols = []
-        execute(sql) do |res|
-          res.nfields.times do |fieldnum|
-            cols << [fieldnum, PG_TYPES[res.ftype(fieldnum)], output_identifier(res.fname(fieldnum))]
-          end
-          @columns = cols.map{|c| c.at(2)}
-          res.ntuples.times do |recnum|
-            converted_rec = {}
-            cols.each do |fieldnum, type_proc, fieldsym|
-              value = res.getvalue(recnum, fieldnum)
-              converted_rec[fieldsym] = (value && type_proc) ? type_proc.call(value) : value
-            end
-            yield converted_rec
-          end
-        end
+      def fetch_rows(sql, &block)
+        return cursor_fetch_rows(sql, &block) if @opts[:cursor]
+        execute(sql){|res| yield_hash_rows(res, fetch_rows_set_cols(res), &block)}
+      end
+      
+      # Uses a cursor for fetching records, instead of fetching the entire result
+      # set at once.  Can be used to process large datasets without holding
+      # all rows in memory (which is what the underlying drivers do
+      # by default). Options:
+      #
+      # * :rows_per_fetch - the number of rows per fetch (default 1000).  Higher
+      #   numbers result in fewer queries but greater memory use.
+      #
+      # Usage:
+      #
+      #   DB[:huge_table].use_cursor.each{|row| p row}
+      #   DB[:huge_table].use_cursor(:rows_per_fetch=>10000).each{|row| p row}
+      #
+      # This is untested with the prepared statement/bound variable support,
+      # and unlikely to work with either.
+      def use_cursor(opts={})
+        clone(:cursor=>{:rows_per_fetch=>1000}.merge(opts))
       end
 
       if SEQUEL_POSTGRES_USES_PG
@@ -334,10 +341,9 @@ module Sequel
           
           protected
           
-          # Return an array of strings for each of the hash values, inserting
-          # them to the correct position in the array.
+          # An array of bound variable values for this query, in the correct order.
           def map_to_prepared_args(hash)
-            @prepared_args.map{|k| hash[k.to_sym].to_s}
+            prepared_args.map{|k| hash[k.to_sym]}
           end
 
           private
@@ -349,11 +355,11 @@ module Sequel
           # elminate ambiguity (and PostgreSQL from raising an exception).
           def prepared_arg(k)
             y, type = k.to_s.split("__")
-            if i = @prepared_args.index(y)
+            if i = prepared_args.index(y)
               i += 1
             else
-              @prepared_args << y
-              i = @prepared_args.length
+              prepared_args << y
+              i = prepared_args.length
             end
             LiteralString.new("#{prepared_arg_placeholder}#{i}#{"::#{type}" if type}")
           end
@@ -407,15 +413,15 @@ module Sequel
         end
         
         # Execute the given type of statement with the hash of values.
-        def call(type, hash, values=nil, &block)
+        def call(type, bind_vars={}, *values, &block)
           ps = to_prepared_statement(type, values)
           ps.extend(BindArgumentMethods)
-          ps.call(hash, &block)
+          ps.call(bind_vars, &block)
         end
         
         # Prepare the given type of statement with the given name, and store
         # it in the database to be called later.
-        def prepare(type, name=nil, values=nil)
+        def prepare(type, name=nil, *values)
           ps = to_prepared_statement(type, values)
           ps.extend(PreparedStatementMethods)
           if name
@@ -435,15 +441,68 @@ module Sequel
       end
       
       private
-
+      
+      # Use a cursor to fetch groups of records at a time, yielding them to the block.
+      def cursor_fetch_rows(sql, &block)
+        server_opts = {:server=>@opts[:server] || :read_only}
+        db.transaction(server_opts) do 
+          begin
+            execute_ddl("DECLARE sequel_cursor NO SCROLL CURSOR WITHOUT HOLD FOR #{sql}", server_opts)
+            rows_per_fetch = @opts[:cursor][:rows_per_fetch].to_i
+            rows_per_fetch = 1000 if rows_per_fetch <= 0
+            fetch_sql = "FETCH FORWARD #{rows_per_fetch} FROM sequel_cursor"
+            cols = nil
+            # Load columns only in the first fetch, so subsequent fetches are faster
+            execute(fetch_sql) do |res|
+              cols = fetch_rows_set_cols(res)
+              yield_hash_rows(res, cols, &block)
+              return if res.ntuples < rows_per_fetch
+            end
+            loop do
+              execute(fetch_sql) do |res|
+                yield_hash_rows(res, cols, &block)
+                return if res.ntuples < rows_per_fetch
+              end
+            end
+          ensure
+            execute_ddl("CLOSE sequel_cursor", server_opts)
+          end
+        end
+      end
+      
+      # Set the @columns based on the result set, and return the array of
+      # field numers, type conversion procs, and name symbol arrays.
+      def fetch_rows_set_cols(res)
+        cols = []
+        res.nfields.times do |fieldnum|
+          cols << [fieldnum, PG_TYPES[res.ftype(fieldnum)], output_identifier(res.fname(fieldnum))]
+        end
+        @columns = cols.map{|c| c.at(2)}
+        cols
+      end
+      
+      # Use the driver's escape_bytea
       def literal_blob(v)
         db.synchronize{|c| "'#{c.escape_bytea(v)}'"}
       end
-
+      
+      # Use the driver's escape_string
       def literal_string(v)
         db.synchronize{|c| "'#{c.escape_string(v)}'"}
       end
-
+      
+      # For each row in the result set, yield a hash with column name symbol
+      # keys and typecasted values.
+      def yield_hash_rows(res, cols)
+        res.ntuples.times do |recnum|
+          converted_rec = {}
+          cols.each do |fieldnum, type_proc, fieldsym|
+            value = res.getvalue(recnum, fieldnum)
+            converted_rec[fieldsym] = (value && type_proc) ? type_proc.call(value) : value
+          end
+          yield converted_rec
+        end
+      end
     end
   end
 end

data/lib/sequel/adapters/shared/mssql.rb

@@ -1,4 +1,5 @@
 module Sequel
+  Dataset::NON_SQL_OPTIONS << :disable_insert_output
   module MSSQL
     module DatabaseMethods
       AUTO_INCREMENT = 'IDENTITY(1,1)'.freeze
@@ -6,7 +7,7 @@ module Sequel
       SERVER_VERSION_SQL = "SELECT CAST(SERVERPROPERTY('ProductVersion') AS varchar)".freeze
       SQL_BEGIN = "BEGIN TRANSACTION".freeze
       SQL_COMMIT = "COMMIT TRANSACTION".freeze
-      SQL_ROLLBACK = "ROLLBACK TRANSACTION".freeze
+      SQL_ROLLBACK = "IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION".freeze
       SQL_ROLLBACK_TO_SAVEPOINT = 'ROLLBACK TRANSACTION autopoint_%d'.freeze
       SQL_SAVEPOINT = 'SAVE TRANSACTION autopoint_%d'.freeze
       TEMPORARY = "#".freeze
@@ -127,7 +128,7 @@ module Sequel
          select(:column_name___column, :data_type___db_type, :character_maximum_length___max_chars, :column_default___default, :is_nullable___allow_null).
          filter(:c__table_name=>m2.call(table_name.to_s))
         if schema = opts[:schema] || default_schema
-          ds.filter!(:table_schema=>schema)
+          ds.filter!(:c__table_schema=>schema)
         end
         ds.map do |row|
           row[:allow_null] = row[:allow_null] == 'YES' ? true : false
@@ -163,6 +164,17 @@ module Sequel
       def type_literal_generic_file(column)
         :image
       end
+
+      # support for clustered index type
+      def index_definition_sql(table_name, index)
+        index_name = index[:name] || default_index_name(table_name, index[:columns])
+        clustered = index[:type] == :clustered
+        if index[:where]
+          raise Error, "Partial indexes are not supported for this database"
+        else
+          "CREATE #{'UNIQUE ' if index[:unique]}#{'CLUSTERED ' if clustered}INDEX #{quote_identifier(index_name)} ON #{quote_schema_table(table_name)} #{literal(index[:columns])}"
+        end
+      end
     end
   
     module DatasetMethods
@@ -176,11 +188,15 @@ module Sequel
       WILDCARD = LiteralString.new('*').freeze
       CONSTANT_MAP = {:CURRENT_DATE=>'CAST(CURRENT_TIMESTAMP AS DATE)'.freeze, :CURRENT_TIME=>'CAST(CURRENT_TIMESTAMP AS TIME)'.freeze}
 
-      # MSSQL uses + for string concatenation
+      # MSSQL uses + for string concatenation, and LIKE is case insensitive by default.
       def complex_expression_sql(op, args)
         case op
         when :'||'
           super(:+, args)
+        when :ILIKE
+          super(:LIKE, args)
+        when :"NOT ILIKE"
+          super(:"NOT LIKE", args)
         else
           super(op, args)
         end
@@ -191,6 +207,16 @@ module Sequel
         CONSTANT_MAP[constant] || super
       end
       
+      # Disable the use of INSERT OUTPUT
+      def disable_insert_output
+        clone(:disable_insert_output=>true)
+      end
+
+      # Disable the use of INSERT OUTPUT, modifying the receiver
+      def disable_insert_output!
+        mutation_method(:disable_insert_output)
+      end
+
       # When returning all rows, if an offset is used, delete the row_number column
       # before yielding the row.
       def fetch_rows(sql, &block)
@@ -201,7 +227,12 @@ module Sequel
       def full_text_search(cols, terms, opts = {})
         filter("CONTAINS (#{literal(cols)}, #{literal(terms)})")
       end
-      
+
+      # Use the OUTPUT clause to get the value of all columns for the newly inserted record.
+      def insert_select(*values)
+        naked.clone(default_server_opts(:sql=>output(nil, [:inserted.*]).insert_sql(*values))).single_record unless opts[:disable_insert_output]
+      end
+
       # MSSQL uses a UNION ALL statement to insert multiple values at once.
       def multi_insert_sql(columns, values)
         [insert_sql(columns, LiteralString.new(values.map {|r| "SELECT #{expression_list(r)}" }.join(" UNION ALL ")))]
@@ -227,9 +258,9 @@ module Sequel
       def output(into, values)
         output = {}
         case values
-          when Hash:
+          when Hash
             output[:column_list], output[:select_list] = values.keys, values.values
-          when Array:
+          when Array
             output[:select_list] = values
         end
         output[:into] = into
@@ -286,6 +317,16 @@ module Sequel
       def supports_is_true?
         false
       end
+      
+      # MSSQL doesn't support JOIN USING
+      def supports_join_using?
+        false
+      end
+
+      # MSSQL does not support multiple columns for the IN/NOT IN operators
+      def supports_multiple_column_in?
+        false
+      end
 
       # MSSQL 2005+ supports window functions
       def supports_window_functions?