sequel 5.20.0 → 5.49.0

data/lib/sequel/adapters/shared/sqlite.rb

@@ -10,6 +10,11 @@ module Sequel
     def self.mock_adapter_setup(db)
       db.instance_exec do
         @sqlite_version = 30903
+
+        def schema_parse_table(*)
+          []
+        end
+        singleton_class.send(:private, :schema_parse_table)
       end
     end
     
@@ -33,6 +38,10 @@ module Sequel
       # booleans be stored as integers, but historically Sequel has used 't'/'f'.
       attr_accessor :integer_booleans
 
+      # Whether to keep CURRENT_TIMESTAMP and similar expressions in UTC.  By
+      # default, the expressions are converted to localtime.
+      attr_accessor :current_timestamp_utc
+
       # A symbol signifying the value of the default transaction mode
       attr_reader :transaction_mode
 
@@ -60,7 +69,7 @@ module Sequel
       def foreign_key_list(table, opts=OPTS)
         m = output_identifier_meth
         h = {}
-        metadata_dataset.with_sql("PRAGMA foreign_key_list(?)", input_identifier_meth.call(table)).each do |row|
+        _foreign_key_list_ds(table).each do |row|
           if r = h[row[:id]]
             r[:columns] << m.call(row[:from])
             r[:key] << m.call(row[:to]) if r[:key]
@@ -173,6 +182,16 @@ module Sequel
 
       private
 
+      # Dataset used for parsing foreign key lists
+      def _foreign_key_list_ds(table)
+        metadata_dataset.with_sql("PRAGMA foreign_key_list(?)", input_identifier_meth.call(table))
+      end
+
+      # Dataset used for parsing schema
+      def _parse_pragma_ds(table_name, opts)
+        metadata_dataset.with_sql("PRAGMA table_#{'x' if sqlite_version > 33100}info(?)", input_identifier_meth(opts[:dataset]).call(table_name))
+      end
+
       # Run all alter_table commands in a transaction.  This is technically only
       # needed for drop column.
       def apply_alter_table(table, ops)
@@ -220,8 +239,12 @@ module Sequel
             super
           end
         when :drop_column
-          ocp = lambda{|oc| oc.delete_if{|c| c.to_s == op[:name].to_s}}
-          duplicate_table(table, :old_columns_proc=>ocp){|columns| columns.delete_if{|s| s[:name].to_s == op[:name].to_s}}
+          if sqlite_version >= 33500
+            super
+          else
+            ocp = lambda{|oc| oc.delete_if{|c| c.to_s == op[:name].to_s}}
+            duplicate_table(table, :old_columns_proc=>ocp){|columns| columns.delete_if{|s| s[:name].to_s == op[:name].to_s}}
+          end
         when :rename_column
           if sqlite_version >= 32500
             super
@@ -238,13 +261,20 @@ module Sequel
         when :drop_constraint
           case op[:type]
           when :primary_key
-            duplicate_table(table){|columns| columns.each{|s| s[:primary_key] = s[:auto_increment] = nil}}
+            duplicate_table(table) do |columns|
+              columns.each do |s|
+                s[:unique] = false if s[:primary_key]
+                s[:primary_key] = s[:auto_increment] = nil
+              end
+            end
           when :foreign_key
             if op[:columns]
               duplicate_table(table, :skip_foreign_key_columns=>op[:columns])
             else
               duplicate_table(table, :no_foreign_keys=>true)
             end
+          when :unique
+            duplicate_table(table, :no_unique=>true)
           else
             duplicate_table(table)
           end
@@ -279,9 +309,15 @@ module Sequel
         false
       end
 
-      # Surround default with parens to appease SQLite
+      # Surround default with parens to appease SQLite.  Add support for GENERATED ALWAYS AS.
       def column_definition_default_sql(sql, column)
         sql << " DEFAULT (#{literal(column[:default])})" if column.include?(:default)
+        if (generated = column[:generated_always_as])
+          if (generated_type = column[:generated_type]) && (generated_type == :stored || generated_type == :virtual)
+            generated_type = generated_type.to_s.upcase
+          end
+          sql << " GENERATED ALWAYS AS (#{literal(generated)}) #{generated_type}"
+        end
       end
     
       # Array of PRAGMA SQL statements based on the Database options that should be applied to
@@ -357,7 +393,7 @@ module Sequel
         old_columns = def_columns.map{|c| c[:name]}
         opts[:old_columns_proc].call(old_columns) if opts[:old_columns_proc]
 
-        yield def_columns if block_given?
+        yield def_columns if defined?(yield)
 
         constraints = (opts[:constraints] || []).dup
         pks = []
@@ -392,15 +428,19 @@ module Sequel
         skip_indexes = []
         indexes(table, :only_autocreated=>true).each do |name, h|
           skip_indexes << name
-          if h[:columns].length == 1 && h[:unique]
-            unique_columns.concat(h[:columns])
+          if h[:unique] && !opts[:no_unique]
+            if h[:columns].length == 1
+              unique_columns.concat(h[:columns])
+            elsif h[:columns].map(&:to_s) != pks
+              constraints << {:type=>:unique, :columns=>h[:columns]}
+            end
           end
         end
         unique_columns -= pks
         unless unique_columns.empty?
           unique_columns.map!{|c| quote_identifier(c)}
           def_columns.each do |c|
-            c[:unique] = true if unique_columns.include?(quote_identifier(c[:name]))
+            c[:unique] = true if unique_columns.include?(quote_identifier(c[:name])) && c[:unique] != false
           end
         end
         
@@ -445,7 +485,16 @@ module Sequel
       # Parse the output of the table_info pragma
       def parse_pragma(table_name, opts)
         pks = 0
-        sch = metadata_dataset.with_sql("PRAGMA table_info(?)", input_identifier_meth(opts[:dataset]).call(table_name)).map do |row|
+        sch = _parse_pragma_ds(table_name, opts).map do |row|
+          if sqlite_version > 33100
+            # table_xinfo PRAGMA used, remove hidden columns
+            # that are not generated columns
+            if row[:generated] = (row.delete(:hidden) != 0)
+              next unless row[:type].end_with?(' GENERATED ALWAYS')
+              row[:type] = row[:type].sub(' GENERATED ALWAYS', '')
+            end
+          end
+
           row.delete(:cid)
           row[:allow_null] = row.delete(:notnull).to_i == 0
           row[:default] = row.delete(:dflt_value)
@@ -462,6 +511,8 @@ module Sequel
           row
         end
 
+        sch.compact!
+
         if pks > 1
           # SQLite does not allow use of auto increment for tables
           # with composite primary keys, so remove auto_increment
@@ -511,10 +562,10 @@ module Sequel
       EXTRACT_MAP = {:year=>"'%Y'", :month=>"'%m'", :day=>"'%d'", :hour=>"'%H'", :minute=>"'%M'", :second=>"'%f'"}.freeze
       EXTRACT_MAP.each_value(&:freeze)
 
-      Dataset.def_sql_method(self, :delete, [['if db.sqlite_version >= 30803', %w'with delete from where'], ["else", %w'delete from where']])
-      Dataset.def_sql_method(self, :insert, [['if db.sqlite_version >= 30803', %w'with insert conflict into columns values on_conflict'], ["else", %w'insert conflict into columns values']])
-      Dataset.def_sql_method(self, :select, [['if opts[:values]', %w'with values compounds'], ['else', %w'with select distinct columns from join where group having compounds order limit lock']])
-      Dataset.def_sql_method(self, :update, [['if db.sqlite_version >= 30803', %w'with update table set where'], ["else", %w'update table set where']])
+      Dataset.def_sql_method(self, :delete, [['if db.sqlite_version >= 33500', %w'with delete from where returning'], ['elsif db.sqlite_version >= 30803', %w'with delete from where'], ["else", %w'delete from where']])
+      Dataset.def_sql_method(self, :insert, [['if db.sqlite_version >= 33500', %w'with insert conflict into columns values on_conflict returning'], ['elsif db.sqlite_version >= 30803', %w'with insert conflict into columns values on_conflict'], ["else", %w'insert conflict into columns values']])
+      Dataset.def_sql_method(self, :select, [['if opts[:values]', %w'with values compounds'], ['else', %w'with select distinct columns from join where group having window compounds order limit lock']])
+      Dataset.def_sql_method(self, :update, [['if db.sqlite_version >= 33500', %w'with update table set from where returning'], ['elsif db.sqlite_version >= 33300', %w'with update table set from where'], ['elsif db.sqlite_version >= 30803', %w'with update table set where'], ["else", %w'update table set where']])
 
       def cast_sql_append(sql, expr, type)
         if type == Time or type == DateTime
@@ -578,7 +629,7 @@ module Sequel
       # SQLite has CURRENT_TIMESTAMP and related constants in UTC instead
       # of in localtime, so convert those constants to local time.
       def constant_sql_append(sql, constant)
-        if c = CONSTANT_MAP[constant]
+        if (c = CONSTANT_MAP[constant]) && !db.current_timestamp_utc
           sql << c
         else
           super
@@ -588,8 +639,8 @@ module Sequel
       # SQLite performs a TRUNCATE style DELETE if no filter is specified.
       # Since we want to always return the count of records, add a condition
       # that is always true and then delete.
-      def delete
-        @opts[:where] ? super : where(1=>1).delete
+      def delete(&block)
+        @opts[:where] ? super : where(1=>1).delete(&block)
       end
       
       # Return an array of strings specifying a query explanation for a SELECT of the
@@ -610,6 +661,21 @@ module Sequel
         super
       end
       
+      # Support insert select for associations, so that the model code can use
+      # returning instead of a separate query.
+      def insert_select(*values)
+        return unless supports_insert_select?
+        # Handle case where query does not return a row
+        server?(:default).with_sql_first(insert_select_sql(*values)) || false
+      end
+
+      # The SQL to use for an insert_select, adds a RETURNING clause to the insert
+      # unless the RETURNING clause is already present.
+      def insert_select_sql(*values)
+        ds = opts[:returning] ? self : returning
+        ds.insert_sql(*values)
+      end
+
       # SQLite uses the nonstandard ` (backtick) for quoting identifiers.
       def quoted_identifier_append(sql, c)
         sql << '`' << c.to_s.gsub('`', '``') << '`'
@@ -691,6 +757,13 @@ module Sequel
         insert_conflict(:ignore)
       end
 
+      # Automatically add aliases to RETURNING values to work around SQLite bug.
+      def returning(*values)
+        return super if values.empty?
+        raise Error, "RETURNING is not supported on #{db.database_type}" unless supports_returning?(:insert)
+        clone(:returning=>_returning_values(values).freeze)
+      end
+
       # SQLite 3.8.3+ supports common table expressions.
       def supports_cte?(type=:select)
         db.sqlite_version >= 30803
@@ -706,6 +779,11 @@ module Sequel
         false
       end
 
+      # SQLite does not support deleting from a joined dataset
+      def supports_deleting_joins?
+        false
+      end
+
       # SQLite does not support INTERSECT ALL or EXCEPT ALL
       def supports_intersect_except_all?
         false
@@ -716,11 +794,21 @@ module Sequel
         false
       end
       
+      # SQLite 3.33.0 supports modifying joined datasets
+      def supports_modifying_joins?
+        db.sqlite_version >= 33300
+      end
+
       # SQLite does not support multiple columns for the IN/NOT IN operators
       def supports_multiple_column_in?
         false
       end
       
+      # SQLite 3.35.0 supports RETURNING on INSERT/UPDATE/DELETE.
+      def supports_returning?(_)
+        db.sqlite_version >= 33500
+      end
+
       # SQLite supports timezones in literal timestamps, since it stores them
       # as text.  But using timezones in timestamps breaks SQLite datetime
       # functions, so we allow the user to override the default per database.
@@ -732,6 +820,11 @@ module Sequel
       def supports_where_true?
         false
       end
+
+      # SQLite 3.28+ supports the WINDOW clause.
+      def supports_window_clause?
+        db.sqlite_version >= 32800
+      end
       
       # SQLite 3.25+ supports window functions.  However, support is only enabled
       # on SQLite 3.26.0+ because internal Sequel usage of window functions
@@ -741,8 +834,28 @@ module Sequel
         db.sqlite_version >= 32600
       end
     
+      # SQLite 3.28.0+ supports all window frame options that Sequel supports
+      def supports_window_function_frame_option?(option)
+        db.sqlite_version >= 32800 ? true : super
+      end
+
       private
       
+      # Add aliases to symbols and identifiers to work around SQLite bug.
+      def _returning_values(values)
+        values.map do |v|
+          case v
+          when Symbol
+            _, c, a = split_symbol(v)
+            a ? v : Sequel.as(v, c)
+          when SQL::Identifier, SQL::QualifiedIdentifier
+            Sequel.as(v, unqualified_column_for(v))
+          else
+            v
+          end
+        end
+      end
+
       # SQLite uses string literals instead of identifiers in AS clauses.
       def as_sql_append(sql, aliaz, column_aliases=nil)
         raise Error, "sqlite does not support derived column lists" if column_aliases
@@ -768,6 +881,13 @@ module Sequel
         end
       end
 
+      # Raise an InvalidOperation exception if insert is not allowed for this dataset.
+      def check_insert_allowed!
+        raise(InvalidOperation, "Grouped datasets cannot be modified") if opts[:group]
+        raise(InvalidOperation, "Joined datasets cannot be modified") if joined_dataset?
+      end
+      alias check_delete_allowed! check_insert_allowed!
+
       # SQLite supports a maximum of 500 rows in a VALUES clause.
       def default_import_slice
         500
@@ -845,9 +965,9 @@ module Sequel
         end
       end
 
-      # SQLite does not natively support NULLS FIRST/LAST.
+      # SQLite supports NULLS FIRST/LAST natively in 3.30+.
       def requires_emulating_nulls_first?
-        true
+        db.sqlite_version < 33000
       end
 
       # SQLite does not support FOR UPDATE, but silently ignore it
@@ -873,6 +993,11 @@ module Sequel
         false
       end
 
+      # SQLite 3.30 supports the FILTER clause for aggregate functions.
+      def supports_filtered_aggregates?
+        db.sqlite_version >= 33000
+      end
+
       # SQLite supports quoted function names.
       def supports_quoted_function_names?
         true
@@ -882,6 +1007,23 @@ module Sequel
       def _truncate_sql(table)
         "DELETE FROM #{table}"
       end
+
+      # Use FROM to specify additional tables in an update query
+      def update_from_sql(sql)
+        if(from = @opts[:from][1..-1]).empty?
+          raise(Error, 'Need multiple FROM tables if updating/deleting a dataset with JOINs') if @opts[:join]
+        else
+          sql << ' FROM '
+          source_list_append(sql, from)
+          select_join_sql(sql)
+        end
+      end
+
+      # Only include the primary table in the main update clause
+      def update_table_sql(sql)
+        sql << ' '
+        source_list_append(sql, @opts[:from][0..0])
+      end
     end
   end
 end

data/lib/sequel/adapters/sqlanywhere.rb

@@ -120,7 +120,7 @@ module Sequel
 
         case type
         when :select
-          yield rs if block_given?
+          yield rs if defined?(yield)
         when :rows
           return @api.sqlany_affected_rows(rs)
         when :insert

data/lib/sequel/adapters/sqlite.rb

@@ -112,7 +112,7 @@ module Sequel
         sqlite3_opts = {}
         sqlite3_opts[:readonly] = typecast_value_boolean(opts[:readonly]) if opts.has_key?(:readonly)
         db = ::SQLite3::Database.new(opts[:database].to_s, sqlite3_opts)
-        db.busy_timeout(opts.fetch(:timeout, 5000))
+        db.busy_timeout(typecast_value_integer(opts.fetch(:timeout, 5000)))
 
         if USE_EXTENDED_RESULT_CODES
           db.extended_result_codes = true

data/lib/sequel/adapters/tinytds.rb

@@ -16,6 +16,19 @@ module Sequel
         c = TinyTds::Client.new(opts)
         c.query_options.merge!(:cache_rows=>false)
 
+        # SEQUEL6: Default to ansi: true
+        if opts[:ansi]
+          sql = %w(
+            ANSI_NULLS
+            ANSI_PADDING
+            ANSI_WARNINGS
+            ANSI_NULL_DFLT_ON
+            QUOTED_IDENTIFIER
+            CONCAT_NULL_YIELDS_NULL
+          ).map{|v| "SET #{v} ON"}.join(";")
+          log_connection_yield(sql, c){c.execute(sql)}
+        end
+
         if (ts = opts[:textsize])
           sql = "SET TEXTSIZE #{typecast_value_integer(ts)}"
           log_connection_yield(sql, c){c.execute(sql)}
@@ -62,7 +75,7 @@ module Sequel
                 return r.public_send(m) if m
               end
             end
-            yield(r) if block_given?
+            yield(r) if defined?(yield)
           rescue TinyTds::Error => e
             raise_error(e, :disconnect=>!c.active?)
           ensure
@@ -133,7 +146,7 @@ module Sequel
 
       # Return true if the :conn argument is present and not active.
       def disconnect_error?(e, opts)
-        super || (opts[:conn] && !opts[:conn].active?) || ((e.is_a?(::TinyTds::Error) && /\A(Attempt to initiate a new Adaptive Server operation with results pending|The request failed to run because the batch is aborted, this can be caused by abort signal sent from client|Adaptive Server connection timed out)/.match(e.message)))
+        super || (opts[:conn] && !opts[:conn].active?) || ((e.is_a?(::TinyTds::Error) && /\A(Attempt to initiate a new Adaptive Server operation with results pending|The request failed to run because the batch is aborted, this can be caused by abort signal sent from client|Adaptive Server connection timed out|DBPROCESS is dead or not enabled)/.match(e.message)))
       end
 
       # Dispose of any possible results of execution.

data/lib/sequel/adapters/utils/mysql_mysql2.rb

@@ -18,6 +18,7 @@ module Sequel
         This connection is still waiting for a result, try again once you have the result
         closed MySQL connection
         The MySQL server is running with the --read-only option so it cannot execute this statement
+        Connection was killed
         END
         # Error messages for mysql and mysql2 that indicate the current connection should be disconnected
         MYSQL_DATABASE_DISCONNECT_ERRORS = /\A#{Regexp.union(disconnect_errors)}/
@@ -55,10 +56,12 @@ module Sequel
             NotNullConstraintViolation
           when 1062
             UniqueConstraintViolation
-          when 1451, 1452
+          when 1451, 1452, 1216, 1217
             ForeignKeyConstraintViolation
           when 4025
             CheckConstraintViolation
+          when 1205
+            DatabaseLockTimeout
           else
             super
           end

data/lib/sequel/ast_transformer.rb

@@ -80,6 +80,12 @@ module Sequel
         SQL::DelayedEvaluation.new(lambda{|ds| v(o.call(ds))})
       when SQL::Wrapper
         SQL::Wrapper.new(v(o.value))
+      when SQL::Expression
+        if o.respond_to?(:sequel_ast_transform)
+          o.sequel_ast_transform(method(:v))
+        else
+          o
+        end
       else
         o
       end

data/lib/sequel/connection_pool/sharded_single.rb

@@ -41,7 +41,10 @@ class Sequel::ShardedSingleConnectionPool < Sequel::ConnectionPool
   # :server :: Should be a symbol specifing the server to disconnect from,
   #            or an array of symbols to specify multiple servers.
   def disconnect(opts=OPTS)
-    (opts[:server] ? Array(opts[:server]) : servers).each{|s| disconnect_server(s)}
+    (opts[:server] ? Array(opts[:server]) : servers).each do |s|
+      raise Sequel::Error, "invalid server: #{s}" unless @servers.has_key?(s)
+      disconnect_server(s)
+    end
   end
 
   def freeze

data/lib/sequel/connection_pool/sharded_threaded.rb

@@ -57,7 +57,7 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # it is yielding all of the connections, which means that until
   # the method's block returns, the pool is locked.
   def all_connections
-    t = Thread.current
+    t = Sequel.current
     sync do
       @allocated.values.each do |threads|
         threads.each do |thread, conn|
@@ -95,9 +95,7 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   #            or an array of symbols to specify multiple servers.
   def disconnect(opts=OPTS)
     (opts[:server] ? Array(opts[:server]) : sync{@servers.keys}).each do |s|
-      if conns = sync{disconnect_server_connections(s)}
-        disconnect_connections(conns)
-      end
+      disconnect_connections(sync{disconnect_server_connections(s)})
     end
   end
   
@@ -121,7 +119,7 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # connection can be acquired, a Sequel::PoolTimeout is raised.
   def hold(server=:default)
     server = pick_server(server)
-    t = Thread.current
+    t = Sequel.current
     if conn = owned_connection(t, server)
       return yield(conn)
     end
@@ -203,9 +201,9 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
 
     until conn = assign_connection(thread, server)
       elapsed = Sequel.elapsed_seconds_since(timer)
+      # :nocov:
       raise_pool_timeout(elapsed, server) if elapsed > timeout
 
-      # :nocov:
       # It's difficult to get to this point, it can only happen if there is a race condition
       # where a connection cannot be acquired even after the thread is signalled by the condition variable
       sync do
@@ -278,13 +276,15 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
   # Mark any allocated connections to be removed when they are checked back in. The calling
   # code should already have the mutex before calling this.
   def disconnect_server_connections(server)
-    @connections_to_remove.concat(allocated(server).values)
+    remove_conns = allocated(server)
+    dis_conns = available_connections(server)
+    raise Sequel::Error, "invalid server: #{server}" unless remove_conns && dis_conns
 
-    if dis_conns = available_connections(server)
-      conns = dis_conns.dup
-      dis_conns.clear
-      @waiters[server].signal
-    end
+    @connections_to_remove.concat(remove_conns.values)
+
+    conns = dis_conns.dup
+    dis_conns.clear
+    @waiters[server].signal
     conns
   end
 

data/lib/sequel/connection_pool/single.rb

@@ -11,7 +11,7 @@ class Sequel::SingleConnectionPool < Sequel::ConnectionPool
 
   # Yield the connection if one has been made.
   def all_connections
-    yield @conn.first if @conn
+    yield @conn.first unless @conn.empty?
   end
 
   # Disconnect the connection from the database.

data/lib/sequel/connection_pool/threaded.rb

@@ -83,7 +83,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   # is available or the timeout expires.  If the timeout expires before a
   # connection can be acquired, a Sequel::PoolTimeout is raised.
   def hold(server=nil)
-    t = Thread.current
+    t = Sequel.current
     if conn = owned_connection(t)
       return yield(conn)
     end
@@ -152,9 +152,9 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
 
     until conn = assign_connection(thread)
       elapsed = Sequel.elapsed_seconds_since(timer)
+      # :nocov:
       raise_pool_timeout(elapsed) if elapsed > timeout
 
-      # :nocov:
       # It's difficult to get to this point, it can only happen if there is a race condition
       # where a connection cannot be acquired even after the thread is signalled by the condition variable
       sync do