sequel 5.39.0 → 5.63.0

data/lib/sequel/adapters/shared/sqlanywhere.rb

@@ -1,5 +1,7 @@
 # frozen-string-literal: true
 
+require_relative '../utils/columns_limit_1'
+
 module Sequel
   module SqlAnywhere
     Sequel::Database.set_shared_adapter_scheme(:sqlanywhere, self)
@@ -193,6 +195,11 @@ module Sequel
         end
       end
 
+      # SQLAnywhere tinyint types are unsigned.
+      def column_schema_tinyint_type_is_unsigned?
+        true
+      end
+
       # SqlAnywhere doesn't support CREATE TABLE AS, it only supports SELECT INTO.
       # Emulating CREATE TABLE AS using SELECT INTO is only possible if a dataset
       # is given as the argument, it can't work with a string, so raise an
@@ -211,6 +218,8 @@ module Sequel
       def schema_column_type(db_type)
         if convert_smallint_to_bool && db_type =~ /smallint/i
           :boolean
+        elsif db_type =~ /unsigned (big)?int/i
+          :integer
         else
           super
         end
@@ -234,6 +243,7 @@ module Sequel
     module DatasetMethods
       Dataset.def_sql_method(self, :insert, %w'insert into columns values')
       Dataset.def_sql_method(self, :select, %w'with select distinct limit columns into from join where group having window compounds order lock')
+      include ::Sequel::Dataset::ColumnsLimit1
 
       # Whether to convert smallint to boolean arguments for this dataset.
       # Defaults to the IBMDB module setting.

data/lib/sequel/adapters/shared/sqlite.rb

@@ -239,8 +239,12 @@ module Sequel
             super
           end
         when :drop_column
-          ocp = lambda{|oc| oc.delete_if{|c| c.to_s == op[:name].to_s}}
-          duplicate_table(table, :old_columns_proc=>ocp){|columns| columns.delete_if{|s| s[:name].to_s == op[:name].to_s}}
+          if sqlite_version >= 33500
+            super
+          else
+            ocp = lambda{|oc| oc.delete_if{|c| c.to_s == op[:name].to_s}}
+            duplicate_table(table, :old_columns_proc=>ocp){|columns| columns.delete_if{|s| s[:name].to_s == op[:name].to_s}}
+          end
         when :rename_column
           if sqlite_version >= 32500
             super
@@ -316,6 +320,11 @@ module Sequel
         end
       end
     
+      # SQLite does not restrict the integer type to a specific range.
+      def column_schema_integer_min_max_values(db_type)
+        nil
+      end
+
       # Array of PRAGMA SQL statements based on the Database options that should be applied to
       # new connections.
       def connection_pragmas
@@ -333,6 +342,11 @@ module Sequel
         ps
       end
 
+      # Support creating STRICT tables via :strict option
+      def create_table_sql(name, generator, options)
+        "#{super}#{' STRICT' if options[:strict]}"
+      end
+
       # SQLite support creating temporary views.
       def create_view_prefix_sql(name, options)
         create_view_sql_append_columns("CREATE #{'TEMPORARY 'if options[:temp]}VIEW #{quote_schema_table(name)}", options[:columns])
@@ -343,6 +357,7 @@ module Sequel
         /foreign key constraint failed\z/i => ForeignKeyConstraintViolation,
         /\A(SQLITE ERROR 275 \(CONSTRAINT_CHECK\) : )?CHECK constraint failed/ => CheckConstraintViolation,
         /\A(SQLITE ERROR 19 \(CONSTRAINT\) : )?constraint failed\z/ => ConstraintViolation,
+        /\Acannot store [A-Z]+ value in [A-Z]+ column / => ConstraintViolation,
         /may not be NULL\z|NOT NULL constraint failed: .+\z/ => NotNullConstraintViolation,
         /\ASQLITE ERROR \d+ \(\) : CHECK constraint failed: / => CheckConstraintViolation
       }.freeze
@@ -389,7 +404,7 @@ module Sequel
         old_columns = def_columns.map{|c| c[:name]}
         opts[:old_columns_proc].call(old_columns) if opts[:old_columns_proc]
 
-        yield def_columns if block_given?
+        yield def_columns if defined?(yield)
 
         constraints = (opts[:constraints] || []).dup
         pks = []
@@ -424,10 +439,10 @@ module Sequel
         skip_indexes = []
         indexes(table, :only_autocreated=>true).each do |name, h|
           skip_indexes << name
-          if h[:unique]
+          if h[:unique] && !opts[:no_unique]
             if h[:columns].length == 1
               unique_columns.concat(h[:columns])
-            elsif h[:columns].map(&:to_s) != pks && !opts[:no_unique]
+            elsif h[:columns].map(&:to_s) != pks
               constraints << {:type=>:unique, :columns=>h[:columns]}
             end
           end
@@ -558,10 +573,10 @@ module Sequel
       EXTRACT_MAP = {:year=>"'%Y'", :month=>"'%m'", :day=>"'%d'", :hour=>"'%H'", :minute=>"'%M'", :second=>"'%f'"}.freeze
       EXTRACT_MAP.each_value(&:freeze)
 
-      Dataset.def_sql_method(self, :delete, [['if db.sqlite_version >= 30803', %w'with delete from where'], ["else", %w'delete from where']])
-      Dataset.def_sql_method(self, :insert, [['if db.sqlite_version >= 30803', %w'with insert conflict into columns values on_conflict'], ["else", %w'insert conflict into columns values']])
+      Dataset.def_sql_method(self, :delete, [['if db.sqlite_version >= 33500', %w'with delete from where returning'], ['elsif db.sqlite_version >= 30803', %w'with delete from where'], ["else", %w'delete from where']])
+      Dataset.def_sql_method(self, :insert, [['if db.sqlite_version >= 33500', %w'with insert conflict into columns values on_conflict returning'], ['elsif db.sqlite_version >= 30803', %w'with insert conflict into columns values on_conflict'], ["else", %w'insert conflict into columns values']])
       Dataset.def_sql_method(self, :select, [['if opts[:values]', %w'with values compounds'], ['else', %w'with select distinct columns from join where group having window compounds order limit lock']])
-      Dataset.def_sql_method(self, :update, [['if db.sqlite_version >= 30803', %w'with update table set where'], ["else", %w'update table set where']])
+      Dataset.def_sql_method(self, :update, [['if db.sqlite_version >= 33500', %w'with update table set from where returning'], ['elsif db.sqlite_version >= 33300', %w'with update table set from where'], ['elsif db.sqlite_version >= 30803', %w'with update table set where'], ["else", %w'update table set where']])
 
       def cast_sql_append(sql, expr, type)
         if type == Time or type == DateTime
@@ -635,8 +650,8 @@ module Sequel
       # SQLite performs a TRUNCATE style DELETE if no filter is specified.
       # Since we want to always return the count of records, add a condition
       # that is always true and then delete.
-      def delete
-        @opts[:where] ? super : where(1=>1).delete
+      def delete(&block)
+        @opts[:where] ? super : where(1=>1).delete(&block)
       end
       
       # Return an array of strings specifying a query explanation for a SELECT of the
@@ -653,10 +668,25 @@ module Sequel
       
       # HAVING requires GROUP BY on SQLite
       def having(*cond)
-        raise(InvalidOperation, "Can only specify a HAVING clause on a grouped dataset") unless @opts[:group]
+        raise(InvalidOperation, "Can only specify a HAVING clause on a grouped dataset") if !@opts[:group] && db.sqlite_version < 33900
         super
       end
       
+      # Support insert select for associations, so that the model code can use
+      # returning instead of a separate query.
+      def insert_select(*values)
+        return unless supports_insert_select?
+        # Handle case where query does not return a row
+        server?(:default).with_sql_first(insert_select_sql(*values)) || false
+      end
+
+      # The SQL to use for an insert_select, adds a RETURNING clause to the insert
+      # unless the RETURNING clause is already present.
+      def insert_select_sql(*values)
+        ds = opts[:returning] ? self : returning
+        ds.insert_sql(*values)
+      end
+
       # SQLite uses the nonstandard ` (backtick) for quoting identifiers.
       def quoted_identifier_append(sql, c)
         sql << '`' << c.to_s.gsub('`', '``') << '`'
@@ -738,6 +768,13 @@ module Sequel
         insert_conflict(:ignore)
       end
 
+      # Automatically add aliases to RETURNING values to work around SQLite bug.
+      def returning(*values)
+        return super if values.empty?
+        raise Error, "RETURNING is not supported on #{db.database_type}" unless supports_returning?(:insert)
+        clone(:returning=>_returning_values(values).freeze)
+      end
+
       # SQLite 3.8.3+ supports common table expressions.
       def supports_cte?(type=:select)
         db.sqlite_version >= 30803
@@ -753,6 +790,11 @@ module Sequel
         false
       end
 
+      # SQLite does not support deleting from a joined dataset
+      def supports_deleting_joins?
+        false
+      end
+
       # SQLite does not support INTERSECT ALL or EXCEPT ALL
       def supports_intersect_except_all?
         false
@@ -763,11 +805,21 @@ module Sequel
         false
       end
       
+      # SQLite 3.33.0 supports modifying joined datasets
+      def supports_modifying_joins?
+        db.sqlite_version >= 33300
+      end
+
       # SQLite does not support multiple columns for the IN/NOT IN operators
       def supports_multiple_column_in?
         false
       end
       
+      # SQLite 3.35.0 supports RETURNING on INSERT/UPDATE/DELETE.
+      def supports_returning?(_)
+        db.sqlite_version >= 33500
+      end
+
       # SQLite supports timezones in literal timestamps, since it stores them
       # as text.  But using timezones in timestamps breaks SQLite datetime
       # functions, so we allow the user to override the default per database.
@@ -800,6 +852,21 @@ module Sequel
 
       private
       
+      # Add aliases to symbols and identifiers to work around SQLite bug.
+      def _returning_values(values)
+        values.map do |v|
+          case v
+          when Symbol
+            _, c, a = split_symbol(v)
+            a ? v : Sequel.as(v, c)
+          when SQL::Identifier, SQL::QualifiedIdentifier
+            Sequel.as(v, unqualified_column_for(v))
+          else
+            v
+          end
+        end
+      end
+
       # SQLite uses string literals instead of identifiers in AS clauses.
       def as_sql_append(sql, aliaz, column_aliases=nil)
         raise Error, "sqlite does not support derived column lists" if column_aliases
@@ -825,6 +892,13 @@ module Sequel
         end
       end
 
+      # Raise an InvalidOperation exception if insert is not allowed for this dataset.
+      def check_insert_allowed!
+        raise(InvalidOperation, "Grouped datasets cannot be modified") if opts[:group]
+        raise(InvalidOperation, "Joined datasets cannot be modified") if joined_dataset?
+      end
+      alias check_delete_allowed! check_insert_allowed!
+
       # SQLite supports a maximum of 500 rows in a VALUES clause.
       def default_import_slice
         500
@@ -944,6 +1018,23 @@ module Sequel
       def _truncate_sql(table)
         "DELETE FROM #{table}"
       end
+
+      # Use FROM to specify additional tables in an update query
+      def update_from_sql(sql)
+        if(from = @opts[:from][1..-1]).empty?
+          raise(Error, 'Need multiple FROM tables if updating/deleting a dataset with JOINs') if @opts[:join]
+        else
+          sql << ' FROM '
+          source_list_append(sql, from)
+          select_join_sql(sql)
+        end
+      end
+
+      # Only include the primary table in the main update clause
+      def update_table_sql(sql)
+        sql << ' '
+        source_list_append(sql, @opts[:from][0..0])
+      end
     end
   end
 end

data/lib/sequel/adapters/sqlanywhere.rb

@@ -120,7 +120,7 @@ module Sequel
 
         case type
         when :select
-          yield rs if block_given?
+          yield rs if defined?(yield)
         when :rows
           return @api.sqlany_affected_rows(rs)
         when :insert

data/lib/sequel/adapters/sqlite.rb

@@ -98,6 +98,11 @@ module Sequel
       # The conversion procs to use for this database
       attr_reader :conversion_procs
 
+      def initialize(opts = OPTS)
+        super
+        @allow_regexp = typecast_value_boolean(opts[:setup_regexp_function])
+      end
+
       # Connect to the database. Since SQLite is a file based database,
       # available options are limited:
       #
@@ -119,6 +124,12 @@ module Sequel
         end
         
         connection_pragmas.each{|s| log_connection_yield(s, db){db.execute_batch(s)}}
+
+        if typecast_value_boolean(opts[:setup_regexp_function])
+          db.create_function("regexp", 2) do |func, regexp_str, string|
+            func.result = Regexp.new(regexp_str).match(string) ? 1 : 0
+          end
+        end
         
         class << db
           attr_reader :prepared_statements
@@ -128,6 +139,12 @@ module Sequel
         db
       end
 
+      # Whether this Database instance is setup to allow regexp matching.
+      # True if the :setup_regexp_function option was passed when creating the Database.
+      def allow_regexp?
+        @allow_regexp
+      end
+
       # Disconnect given connections from the database.
       def disconnect_connection(c)
         c.prepared_statements.each_value{|v| v.first.close}
@@ -189,26 +206,24 @@ module Sequel
       # Yield an available connection.  Rescue
       # any SQLite3::Exceptions and turn them into DatabaseErrors.
       def _execute(type, sql, opts, &block)
-        begin
-          synchronize(opts[:server]) do |conn|
-            return execute_prepared_statement(conn, type, sql, opts, &block) if sql.is_a?(Symbol)
-            log_args = opts[:arguments]
-            args = {}
-            opts.fetch(:arguments, OPTS).each{|k, v| args[k] = prepared_statement_argument(v)}
-            case type
-            when :select
-              log_connection_yield(sql, conn, log_args){conn.query(sql, args, &block)}
-            when :insert
-              log_connection_yield(sql, conn, log_args){conn.execute(sql, args)}
-              conn.last_insert_row_id
-            when :update
-              log_connection_yield(sql, conn, log_args){conn.execute_batch(sql, args)}
-              conn.changes
-            end
+        synchronize(opts[:server]) do |conn|
+          return execute_prepared_statement(conn, type, sql, opts, &block) if sql.is_a?(Symbol)
+          log_args = opts[:arguments]
+          args = {}
+          opts.fetch(:arguments, OPTS).each{|k, v| args[k] = prepared_statement_argument(v)}
+          case type
+          when :select
+            log_connection_yield(sql, conn, log_args){conn.query(sql, args, &block)}
+          when :insert
+            log_connection_yield(sql, conn, log_args){conn.execute(sql, args)}
+            conn.last_insert_row_id
+          when :update
+            log_connection_yield(sql, conn, log_args){conn.execute_batch(sql, args)}
+            conn.changes
           end
-        rescue SQLite3::Exception => e
-          raise_error(e)
         end
+      rescue SQLite3::Exception => e
+        raise_error(e)
       end
       
       # The SQLite adapter does not need the pool to convert exceptions.
@@ -323,6 +338,28 @@ module Sequel
       BindArgumentMethods = prepared_statements_module(:bind, ArgumentMapper)
       PreparedStatementMethods = prepared_statements_module(:prepare, BindArgumentMethods)
 
+      # Support regexp functions if using :setup_regexp_function Database option.
+      def complex_expression_sql_append(sql, op, args)
+        case op
+        when :~, :'!~', :'~*', :'!~*'
+          return super unless supports_regexp?
+
+          case_insensitive = [:'~*', :'!~*'].include?(op)
+          sql << 'NOT ' if [:'!~', :'!~*'].include?(op)
+          sql << '('
+          sql << 'LOWER(' if case_insensitive
+          literal_append(sql, args[0])
+          sql << ')' if case_insensitive
+          sql << ' REGEXP '
+          sql << 'LOWER(' if case_insensitive
+          literal_append(sql, args[1])
+          sql << ')' if case_insensitive
+          sql << ')'
+        else
+          super
+        end
+      end
+
       def fetch_rows(sql)
         execute(sql) do |result|
           cps = db.conversion_procs
@@ -346,6 +383,11 @@ module Sequel
           end
         end
       end
+
+      # Support regexp if using :setup_regexp_function Database option.
+      def supports_regexp?
+        db.allow_regexp?
+      end
       
       private
       

data/lib/sequel/adapters/tinytds.rb

@@ -75,7 +75,7 @@ module Sequel
                 return r.public_send(m) if m
               end
             end
-            yield(r) if block_given?
+            yield(r) if defined?(yield)
           rescue TinyTds::Error => e
             raise_error(e, :disconnect=>!c.active?)
           ensure

data/lib/sequel/adapters/utils/columns_limit_1.rb

@@ -0,0 +1,22 @@
+# frozen-string-literal: true
+
+module Sequel
+  class Dataset
+    module ColumnsLimit1
+      COLUMNS_CLONE_OPTIONS = {:distinct => nil, :limit => 1, :offset=>nil, :where=>nil, :having=>nil, :order=>nil, :row_proc=>nil, :graph=>nil, :eager_graph=>nil}.freeze
+
+      # Use a limit of 1 instead of a limit of 0 when
+      # getting the columns.
+      def columns!
+        ds = clone(COLUMNS_CLONE_OPTIONS)
+        ds.each{break}
+
+        if cols = ds.cache[:_columns]
+          self.columns = cols
+        else
+          []
+        end
+      end
+    end
+  end
+end

data/lib/sequel/adapters/utils/mysql_mysql2.rb

@@ -34,7 +34,7 @@ module Sequel
         def execute(sql, opts=OPTS, &block)
           if opts[:sproc]
             call_sproc(sql, opts, &block)
-          elsif sql.is_a?(Symbol)
+          elsif sql.is_a?(Symbol) || sql.is_a?(Sequel::Dataset::ArgumentMapper)
             execute_prepared_statement(sql, opts, &block)
           else
             synchronize(opts[:server]){|conn| _execute(conn, sql, opts, &block)}

data/lib/sequel/ast_transformer.rb

@@ -80,6 +80,12 @@ module Sequel
         SQL::DelayedEvaluation.new(lambda{|ds| v(o.call(ds))})
       when SQL::Wrapper
         SQL::Wrapper.new(v(o.value))
+      when SQL::Expression
+        if o.respond_to?(:sequel_ast_transform)
+          o.sequel_ast_transform(method(:v))
+        else
+          o
+        end
       else
         o
       end

data/lib/sequel/connection_pool/sharded_single.rb

@@ -55,13 +55,11 @@ class Sequel::ShardedSingleConnectionPool < Sequel::ConnectionPool
   # Yields the connection to the supplied block for the given server.
   # This method simulates the ConnectionPool#hold API.
   def hold(server=:default)
-    begin
-      server = pick_server(server)
-      yield(@conns[server] ||= make_new(server))
-    rescue Sequel::DatabaseDisconnectError, *@error_classes => e
-      disconnect_server(server) if disconnect_error?(e)
-      raise
-    end
+    server = pick_server(server)
+    yield(@conns[server] ||= make_new(server))
+  rescue Sequel::DatabaseDisconnectError, *@error_classes => e
+    disconnect_server(server) if disconnect_error?(e)
+    raise
   end
   
   # The ShardedSingleConnectionPool always has a maximum size of 1.

data/lib/sequel/connection_pool/sharded_threaded.rb

@@ -22,6 +22,8 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
     @connections_to_disconnect = []
     @servers = opts.fetch(:servers_hash, Hash.new(:default))
     remove_instance_variable(:@waiter)
+    remove_instance_variable(:@allocated)
+    @allocated = {}
     @waiters = {}
 
     add_servers([:default])
@@ -36,7 +38,9 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
         unless @servers.has_key?(server)
           @servers[server] = server
           @available_connections[server] = []
-          @allocated[server] = {}
+          allocated = {}
+          allocated.compare_by_identity
+          @allocated[server] = allocated
           @waiters[server] = ConditionVariable.new
         end
       end

data/lib/sequel/connection_pool/single.rb

@@ -24,15 +24,13 @@ class Sequel::SingleConnectionPool < Sequel::ConnectionPool
 
   # Yield the connection to the block.
   def hold(server=nil)
-    begin
-      unless c = @conn.first
-        @conn.replace([c = make_new(:default)])
-      end
-      yield c
-    rescue Sequel::DatabaseDisconnectError, *@error_classes => e
-      disconnect if disconnect_error?(e)
-      raise
+    unless c = @conn.first
+      @conn.replace([c = make_new(:default)])
     end
+    yield c
+  rescue Sequel::DatabaseDisconnectError, *@error_classes => e
+    disconnect if disconnect_error?(e)
+    raise
   end
 
   # The SingleConnectionPool always has a maximum size of 1.

data/lib/sequel/connection_pool/threaded.rb

@@ -3,7 +3,7 @@
 # A connection pool allowing multi-threaded access to a pool of connections.
 # This is the default connection pool used by Sequel.
 class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
-  USE_WAITER = true
+  USE_WAITER = true # SEQUEL6: Remove
   Sequel::Deprecation.deprecate_constant(self, :USE_WAITER)
 
   # The maximum number of connections this pool will create (per shard/server
@@ -12,17 +12,17 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   
   # An array of connections that are available for use by the pool.
   # The calling code should already have the mutex before calling this.
-  attr_reader :available_connections
+  attr_reader :available_connections # SEQUEL6: Remove
   
-  # A hash with thread keys and connection values for currently allocated connections.
+  # A hash with thread/fiber keys and connection values for currently allocated connections.
   # The calling code should already have the mutex before calling this.
-  attr_reader :allocated
+  attr_reader :allocated # SEQUEL6: Remove
 
   # The following additional options are respected:
   # :max_connections :: The maximum number of connections the connection pool
   #                     will open (default 4)
   # :pool_timeout :: The amount of seconds to wait to acquire a connection
-  #                  before raising a PoolTimeoutError (default 5)
+  #                  before raising a PoolTimeout error (default 5)
   def initialize(db, opts = OPTS)
     super
     @max_size = Integer(opts[:max_connections] || 4)
@@ -31,6 +31,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     @connection_handling = opts[:connection_handling]
     @available_connections = []
     @allocated = {}
+    @allocated.compare_by_identity
     @timeout = Float(opts[:pool_timeout] || 5)
     @waiter = ConditionVariable.new
   end
@@ -49,8 +50,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
     end
   end
   
-  # Removes all connections currently available, optionally
-  # yielding each connection to the given block. This method has the effect of 
+  # Removes all connections currently available. This method has the effect of 
   # disconnecting from the database, assuming that no connections are currently
   # being used.  If you want to be able to disconnect connections that are
   # currently in use, use the ShardedThreadedConnectionPool, which can do that.
@@ -134,7 +134,7 @@ class Sequel::ThreadedConnectionPool < Sequel::ConnectionPool
   # calling this.
   #
   # This should return a connection is one is available within the timeout,
-  # or nil if a connection could not be acquired within the timeout.
+  # or raise PoolTimeout if a connection could not be acquired within the timeout.
   def acquire(thread)
     if conn = assign_connection(thread)
       return conn