sequel 5.2.0 → 5.3.0

data/lib/sequel/adapters/shared/sqlite.rb

@@ -641,6 +641,11 @@ module Sequel
         db.sqlite_version >= 30803
       end
 
+      # SQLite supports CTEs in subqueries if it supports CTEs.
+      def supports_cte_in_subqueries?
+        supports_cte?
+      end
+
       # SQLite does not support table aliases with column aliases
       def supports_derived_column_lists?
         false
@@ -765,6 +770,11 @@ module Sequel
         expression_list_append(sql, opts[:values])
       end
 
+      # SQLite does not support CTEs directly inside UNION/INTERSECT/EXCEPT.
+      def supports_cte_in_compounds?
+        false
+      end
+
       # SQLite supports quoted function names.
       def supports_quoted_function_names?
         true

data/lib/sequel/adapters/utils/mysql_mysql2.rb

@@ -56,6 +56,8 @@ module Sequel
             UniqueConstraintViolation
           when 1451, 1452
             ForeignKeyConstraintViolation
+          when 4025
+            CheckConstraintViolation
           else
             super
           end

data/lib/sequel/connection_pool.rb

@@ -76,6 +76,9 @@ class Sequel::ConnectionPool
   # connection made, and is usually used to set custom per-connection settings.
   attr_accessor :after_connect
 
+  # An array of sql strings to execute on each new connection.
+  attr_accessor :connect_sqls
+
   # The Sequel::Database object tied to this connection pool.
   attr_accessor :db
 
@@ -86,6 +89,7 @@ class Sequel::ConnectionPool
   # :after_connect :: A callable object called after each new connection is made, with the
   #                   connection object (and server argument if the callable accepts 2 arguments),
   #                   useful for customizations that you want to apply to all connections.
+  # :connect_sqls :: An array of sql strings to execute on each new connection, after :after_connect runs.
   # :preconnect :: Automatically create the maximum number of connections, so that they don't
   #                need to be created as needed.  This is useful when connecting takes a long time
   #                and you want to avoid possible latency during runtime.
@@ -94,6 +98,7 @@ class Sequel::ConnectionPool
   def initialize(db, opts=OPTS)
     @db = db
     @after_connect = opts[:after_connect]
+    @connect_sqls = opts[:connect_sqls]
     @error_classes = db.send(:database_error_classes).dup.freeze
   end
   
@@ -119,6 +124,7 @@ class Sequel::ConnectionPool
   def make_new(server)
     begin
       conn = @db.connect(server)
+
       if ac = @after_connect
         if ac.arity == 2
           ac.call(conn, server)
@@ -126,6 +132,12 @@ class Sequel::ConnectionPool
           ac.call(conn)
         end
       end
+
+      if cs = @connect_sqls
+        cs.each do |sql|
+          db.send(:log_connection_execute, conn, sql)
+        end
+      end
     rescue Exception=>exception
       raise Sequel.convert_exception_class(exception, Sequel::DatabaseConnectionError)
     end

data/lib/sequel/database/misc.rb

@@ -151,6 +151,19 @@ module Sequel
         concurrent = typecast_value_string(@opts[:preconnect]) == "concurrently"
         @pool.send(:preconnect, concurrent)
       end
+
+      case exts = @opts[:extensions]
+      when String
+        extension(*exts.split(',').map(&:to_sym))
+      when Array
+        extension(*exts)
+      when Symbol
+        extension(exts)
+      when nil
+        # nothing
+      else
+        raise Error, "unsupported Database :extensions option: #{@opts[:extensions].inspect}"
+      end
     end
 
     # Freeze internal data structures for the Database instance.

data/lib/sequel/dataset/dataset_module.rb

@@ -20,7 +20,7 @@ module Sequel
       meths = (<<-METHS).split.map(&:to_sym)
         where exclude exclude_having having
         distinct grep group group_and_count group_append 
-        limit offset order order_append order_prepend 
+        limit offset order order_append order_prepend reverse 
         select select_all select_append select_group server
       METHS
 

data/lib/sequel/dataset/features.rb

@@ -192,6 +192,11 @@ module Sequel
       true
     end
 
+    # Whether common table expressions are supported in UNION/INTERSECT/EXCEPT clauses.
+    def supports_cte_in_compounds?
+      supports_cte_in_subqueries?
+    end
+
     # Whether the database supports quoting function names.
     def supports_quoted_function_names?
       false

data/lib/sequel/dataset/query.rb

@@ -774,6 +774,16 @@ module Sequel
     #   DB[:items].returning # RETURNING *
     #   DB[:items].returning(nil) # RETURNING NULL
     #   DB[:items].returning(:id, :name) # RETURNING id, name
+    #
+    #   DB[:items].returning.insert(:a=>1) do |hash|
+    #     # hash for each row inserted, with values for all columns
+    #   end
+    #   DB[:items].returning.update(:a=>1) do |hash|
+    #     # hash for each row updated, with values for all columns
+    #   end
+    #   DB[:items].returning.delete(:a=>1) do |hash|
+    #     # hash for each row deleted, with values for all columns
+    #   end
     def returning(*values)
       raise Error, "RETURNING is not supported on #{db.database_type}" unless supports_returning?(:insert)
       clone(:returning=>values.freeze)
@@ -1122,7 +1132,7 @@ module Sequel
 
     # Add the dataset to the list of compounds
     def compound_clone(type, dataset, opts)
-      if hoist_cte?(dataset)
+      if dataset.is_a?(Dataset) && dataset.opts[:with] && !supports_cte_in_compounds?
         s, ds = hoist_cte(dataset)
         return s.compound_clone(type, ds, opts)
       end
@@ -1199,6 +1209,12 @@ module Sequel
         cond = filter_expr(cond, &block)
         cond = SQL::BooleanExpression.invert(cond) if invert
         cond = SQL::BooleanExpression.new(combine, @opts[clause], cond) if @opts[clause]
+
+        if cond.nil?
+          Sequel::Deprecation.deprecate('Filtering method called on dataset with no existing filter with virtual row block and no argument and virtual row block returned nil.  Currently, this results in the filter being ignored instead of using a NULL filter.  In Sequel 5.4+, this behavior will change to using a NULL filter, similar to the behavior in all other cases.')
+          #cond = Sequel::NULL
+        end
+
         clone(clause => cond)
       end
     end
@@ -1213,11 +1229,9 @@ module Sequel
       expr = nil if expr == EMPTY_ARRAY
 
       if block
-        if expr
-          return SQL::BooleanExpression.new(:AND, filter_expr(expr), filter_expr(Sequel.virtual_row(&block)))
-        else
-          return filter_expr(Sequel.virtual_row(&block))
-        end
+        cond = filter_expr(Sequel.virtual_row(&block))
+        cond = SQL::BooleanExpression.new(:AND, filter_expr(expr), cond) if expr
+        return cond
       end
 
       case expr

data/lib/sequel/dataset/sql.rb

@@ -49,6 +49,9 @@ module Sequel
 
       if values.is_a?(Array) && values.empty? && !insert_supports_empty_values? 
         columns, values = insert_empty_columns_values
+      elsif values.is_a?(Dataset) && hoist_cte?(values) && supports_cte?(:insert)
+        ds, values = hoist_cte(values)
+        return ds.clone(:columns=>columns, :values=>values).send(:_insert_sql)
       end
       clone(:columns=>columns, :values=>values).send(:_insert_sql)
     end

data/lib/sequel/extensions/pg_extended_date_support.rb

@@ -27,6 +27,8 @@ module Sequel
       TIME_YEAR_1 = Time.at(-62135596800).utc
       INFINITE_TIMESTAMP_STRINGS = ['infinity'.freeze, '-infinity'.freeze].freeze
       INFINITE_DATETIME_VALUES = ([PLUS_INFINITY, MINUS_INFINITY] + INFINITE_TIMESTAMP_STRINGS).freeze
+      PLUS_DATE_INFINITY = Date::Infinity.new
+      MINUS_DATE_INFINITY = -PLUS_DATE_INFINITY
 
       # Add dataset methods and update the conversion proces for dates and timestamps.
       def self.extended(db)
@@ -52,6 +54,8 @@ module Sequel
           :nil
         when 'string'
           :string
+        when 'date'
+          :date
         when 'float'
           :float
         when String, true
@@ -111,6 +115,8 @@ module Sequel
           nil
         when :string
           value
+        when :date
+          value == 'infinity' ? PLUS_DATE_INFINITY : MINUS_DATE_INFINITY
         else
           value == 'infinity' ? PLUS_INFINITY : MINUS_INFINITY
         end
@@ -173,6 +179,15 @@ module Sequel
           end
         end
 
+        # Handle Date::Infinity values
+        def literal_other_append(sql, v)
+          if v.is_a?(Date::Infinity)
+            sql << (v > 0 ? "'infinity'" : "'-infinity'")
+          else
+            super
+          end
+        end
+
         if RUBY_ENGINE == 'jruby'
           # :nocov:
 

data/lib/sequel/extensions/synchronize_sql.rb

@@ -0,0 +1,45 @@
+# frozen-string-literal: true
+#
+# The synchronize_sql extension checks out a connection from the pool while
+# generating an SQL string.  In cases where a connection is necessary
+# in order to properly escape input, and multiple inputs in the query need
+# escaping, this can result in fewer connection checkouts and better
+# overall performance. In other cases this results in a performance decrease
+# because a connection is checked out and either not used or kept checked out
+# longer than necessary.
+#
+# The adapters where this extension may improve performance include amalgalite,
+# mysql2, postgres, jdbc/postgresql, and tinytds. In these adapters, escaping
+# strings requires a connection object for as proper escaping requires calling
+# an escaping method on the connection object.
+#
+# This extension is most helpful when dealing with queries with lots of
+# strings that need escaping (e.g. IN queries with long lists).  By default,
+# a connection will be checked out and back in for each string to be escaped,
+# which under high contention can cause the query to spend longer generating
+# the SQL string than the actual pool timeout (since every individual checkout
+# will take less than the timeout, but the sum of all of them can be greater).
+#
+# This extension is unnecessary and will decrease performance if the single
+# threaded connection pool is used.
+
+#
+module Sequel
+  class Dataset
+    module SynchronizeSQL
+      %w'insert select update delete'.each do |type|
+        define_method(:"#{type}_sql") do |*args|
+          if @opts[:sql].is_a?(String)
+            return super(*args)
+          end
+
+          db.synchronize(@opts[:server]) do
+            super(*args)
+          end
+        end
+      end
+    end
+
+    register_extension(:synchronize_sql, SynchronizeSQL)
+  end
+end

data/lib/sequel/model/associations.rb

@@ -145,6 +145,7 @@ module Sequel
           rn = ds.row_number_column 
           limit, offset = limit_and_offset
           ds = ds.unordered.select_append{|o| o.row_number.function.over(:partition=>predicate_key, :order=>ds.opts[:order]).as(rn)}.from_self
+          ds = ds.order(rn) if ds.db.database_type == :mysql
           ds = if !returns_array?
             ds.where(rn => offset ? offset+1 : 1)
           elsif offset

data/lib/sequel/model/base.rb

@@ -1498,22 +1498,15 @@ module Sequel
           model.default_set_fields_options
         end
 
-        case opts[:missing]
-        when :skip
+        case missing = opts[:missing]
+        when :skip, :raise
+          do_raise = true if missing == :raise
           fields.each do |f|
             if hash.has_key?(f) 
               set_column_value("#{f}=", hash[f])
             elsif f.is_a?(Symbol) && hash.has_key?(sf = f.to_s)
               set_column_value("#{sf}=", hash[sf])
-            end
-          end
-        when :raise
-          fields.each do |f|
-            if hash.has_key?(f)
-              set_column_value("#{f}=", hash[f])
-            elsif f.is_a?(Symbol) && hash.has_key?(sf = f.to_s)
-              set_column_value("#{sf}=", hash[sf])
-            else
+            elsif do_raise
               raise(Sequel::Error, "missing field in hash: #{f.inspect} not in #{hash.inspect}")
             end
           end

data/lib/sequel/plugins/validation_helpers.rb

@@ -48,9 +48,9 @@ module Sequel
     #     def default_validation_helpers_options(type)
     #       case type
     #       when :exact_length
-    #         {message: lambda{|exact| I18n.t("errors.exact_length", exact: exact)}
+    #         {message: lambda{|exact| I18n.t("errors.exact_length", exact: exact)}}
     #       when :integer
-    #         {message: lambda{I18n.t("errors.integer")}
+    #         {message: lambda{I18n.t("errors.integer")}}
     #       else
     #         super
     #       end

data/lib/sequel/version.rb

@@ -5,7 +5,7 @@ module Sequel
   MAJOR = 5
   # The minor version of Sequel.  Bumped for every non-patch level
   # release, generally around once a month.
-  MINOR = 2
+  MINOR = 3
   # The tiny version of Sequel.  Usually 0, only bumped for bugfix
   # releases that fix regressions from previous versions.
   TINY  = 0

data/spec/adapters/postgres_spec.rb

@@ -225,14 +225,6 @@ describe "PostgreSQL", 'INSERT ON CONFLICT' do
     @ds.insert_conflict(:constraint=>:ic_test_a_uidx, :update=>{:b=>6}, :update_where=>{Sequel[:ic_test][:b]=>4}).insert(1, 3, 4).must_be_nil
     @ds.all.must_equal [{:a=>1, :b=>5, :c=>5, :c_is_unique=>false}]
   end
-
-  it "Dataset#insert_conflict should respect expressions in the target argument" do
-    @ds.insert_conflict(:target=>:a).insert_sql(1, 2, 3).must_equal "INSERT INTO \"ic_test\" VALUES (1, 2, 3) ON CONFLICT (\"a\") DO NOTHING"
-    @ds.insert_conflict(:target=>:c, :conflict_where=>{:c_is_unique=>true}).insert_sql(1, 2, 3).must_equal "INSERT INTO \"ic_test\" VALUES (1, 2, 3) ON CONFLICT (\"c\") WHERE (\"c_is_unique\" IS TRUE) DO NOTHING"
-    @ds.insert_conflict(:target=>[:b, :c]).insert_sql(1, 2, 3).must_equal "INSERT INTO \"ic_test\" VALUES (1, 2, 3) ON CONFLICT (\"b\", \"c\") DO NOTHING"
-    @ds.insert_conflict(:target=>[:b, Sequel.function(:round, :c)]).insert_sql(1, 2, 3).must_equal "INSERT INTO \"ic_test\" VALUES (1, 2, 3) ON CONFLICT (\"b\", round(\"c\")) DO NOTHING"
-    @ds.insert_conflict(:target=>[:b, Sequel.virtual_row{|o| o.round(:c)}]).insert_sql(1, 2, 3).must_equal "INSERT INTO \"ic_test\" VALUES (1, 2, 3) ON CONFLICT (\"b\", round(\"c\")) DO NOTHING"
-  end
 end if DB.server_version >= 90500
 
 describe "A PostgreSQL database" do
@@ -355,13 +347,6 @@ describe "A PostgreSQL database" do
     Sequel.connect(DB.opts.merge(:notice_receiver=>proc{|r| a = r.result_error_message})){|db| db.do("BEGIN\nRAISE WARNING 'foo';\nEND;")}
     a.must_equal "WARNING:  foo\n"
   end if uses_pg && DB.server_version >= 90000
-
-  # These only test the SQL created, because a true test using file_fdw or postgres_fdw
-  # requires superuser permissions, and you should not be running the tests as a superuser.
-  it "should support creating and dropping foreign tables" do
-    DB.send(:create_table_sql, :t, DB.create_table_generator{Integer :a}, :foreign=>:f, :options=>{:o=>1}).must_equal 'CREATE FOREIGN TABLE "t" ("a" integer) SERVER "f" OPTIONS (o \'1\')'
-    DB.send(:drop_table_sql, :t, :foreign=>true).must_equal 'DROP FOREIGN TABLE "t"'
-  end
 end
 
 describe "A PostgreSQL database with domain types" do
@@ -787,6 +772,8 @@ describe "A PostgreSQL dataset with a timestamp field" do
     @db[:test3].get(:time).must_be_nil
     @db.convert_infinite_timestamps = 'string'
     @db[:test3].get(:time).must_equal 'infinity'
+    @db.convert_infinite_timestamps = 'date'
+    @db[:test3].get(:time).must_equal Date::Infinity.new
     @db.convert_infinite_timestamps = 'float'
     @db[:test3].get(:time).must_equal 1.0/0.0
     @db.convert_infinite_timestamps = 't'
@@ -805,6 +792,8 @@ describe "A PostgreSQL dataset with a timestamp field" do
     @db[:test3].get(:time).must_be_nil
     @db.convert_infinite_timestamps = :string
     @db[:test3].get(:time).must_equal '-infinity'
+    @db.convert_infinite_timestamps = :date
+    @db[:test3].get(:time).must_equal -Date::Infinity.new
     @db.convert_infinite_timestamps = :float
     @db[:test3].get(:time).must_equal(-1.0/0.0)
   end
@@ -1527,47 +1516,34 @@ describe "Postgres::Database functions, languages, schemas, and triggers" do
 
   it "#create_function and #drop_function should create and drop functions" do
     proc{@d['SELECT tf()'].all}.must_raise(Sequel::DatabaseError)
-    args = ['tf', 'SELECT 1', {:returns=>:integer}]
-    @d.send(:create_function_sql, *args).must_match(/\A\s*CREATE FUNCTION tf\(\)\s+RETURNS integer\s+LANGUAGE SQL\s+AS 'SELECT 1'\s*\z/)
-    @d.create_function(*args)
+    @d.create_function('tf', 'SELECT 1', :returns=>:integer)
     @d['SELECT tf()'].all.must_equal [{:tf=>1}]
-    @d.send(:drop_function_sql, 'tf').must_equal 'DROP FUNCTION tf()'
     @d.drop_function('tf')
     proc{@d['SELECT tf()'].all}.must_raise(Sequel::DatabaseError)
   end
 
   it "#create_function and #drop_function should support options" do
     args = ['tf', 'SELECT $1 + $2', {:args=>[[:integer, :a], :integer], :replace=>true, :returns=>:integer, :language=>'SQL', :behavior=>:immutable, :strict=>true, :security_definer=>true, :cost=>2, :set=>{:search_path => 'public'}}]
-    @d.send(:create_function_sql,*args).must_match(/\A\s*CREATE OR REPLACE FUNCTION tf\(a integer, integer\)\s+RETURNS integer\s+LANGUAGE SQL\s+IMMUTABLE\s+STRICT\s+SECURITY DEFINER\s+COST 2\s+SET search_path = public\s+AS 'SELECT \$1 \+ \$2'\s*\z/)
     @d.create_function(*args)
     # Make sure replace works
     @d.create_function(*args)
     @d['SELECT tf(1, 2)'].all.must_equal [{:tf=>3}]
     args = ['tf', {:if_exists=>true, :cascade=>true, :args=>[[:integer, :a], :integer]}]
-    @d.send(:drop_function_sql,*args).must_equal 'DROP FUNCTION IF EXISTS tf(a integer, integer) CASCADE'
     @d.drop_function(*args)
     # Make sure if exists works
     @d.drop_function(*args)
   end
 
   it "#create_language and #drop_language should create and drop languages" do
-    @d.send(:create_language_sql, :plpgsql).must_equal 'CREATE LANGUAGE plpgsql'
     @d.create_language(:plpgsql, :replace=>true) if @d.server_version < 90000
     proc{@d.create_language(:plpgsql)}.must_raise(Sequel::DatabaseError)
-    @d.send(:drop_language_sql, :plpgsql).must_equal 'DROP LANGUAGE plpgsql'
     @d.drop_language(:plpgsql) if @d.server_version < 90000
     proc{@d.drop_language(:plpgsql)}.must_raise(Sequel::DatabaseError) if @d.server_version < 90000
-    @d.send(:create_language_sql, :plpgsql, :replace=>true, :trusted=>true, :handler=>:a, :validator=>:b).must_equal(@d.server_version >= 90000 ? 'CREATE OR REPLACE TRUSTED LANGUAGE plpgsql HANDLER a VALIDATOR b' : 'CREATE TRUSTED LANGUAGE plpgsql HANDLER a VALIDATOR b')
-    @d.send(:drop_language_sql, :plpgsql, :if_exists=>true, :cascade=>true).must_equal 'DROP LANGUAGE IF EXISTS plpgsql CASCADE'
     # Make sure if exists works
     @d.drop_language(:plpgsql, :if_exists=>true, :cascade=>true) if @d.server_version < 90000
   end
 
   it "#create_schema and #drop_schema should create and drop schemas" do
-    @d.send(:create_schema_sql, :sequel).must_equal 'CREATE SCHEMA "sequel"'
-    @d.send(:create_schema_sql, :sequel, :if_not_exists=>true, :owner=>:foo).must_equal 'CREATE SCHEMA IF NOT EXISTS "sequel" AUTHORIZATION "foo"'
-    @d.send(:drop_schema_sql, :sequel).must_equal 'DROP SCHEMA "sequel"'
-    @d.send(:drop_schema_sql, :sequel, :if_exists=>true, :cascade=>true).must_equal 'DROP SCHEMA IF EXISTS "sequel" CASCADE'
     @d.create_schema(:sequel)
     @d.create_schema(:sequel, :if_not_exists=>true) if @d.server_version >= 90300
     @d.create_table(Sequel[:sequel][:test]){Integer :a}
@@ -1577,7 +1553,6 @@ describe "Postgres::Database functions, languages, schemas, and triggers" do
   it "#create_trigger and #drop_trigger should create and drop triggers" do
     @d.create_language(:plpgsql) if @d.server_version < 90000
     @d.create_function(:tf, 'BEGIN IF NEW.value IS NULL THEN RAISE EXCEPTION \'Blah\'; END IF; RETURN NEW; END;', :language=>:plpgsql, :returns=>:trigger)
-    @d.send(:create_trigger_sql, :test, :identity, :tf, :each_row=>true).must_equal 'CREATE TRIGGER identity BEFORE INSERT OR UPDATE OR DELETE ON "test" FOR EACH ROW EXECUTE PROCEDURE tf()'
     @d.create_table(:test){String :name; Integer :value}
     @d.create_trigger(:test, :identity, :tf, :each_row=>true)
     @d[:test].insert(:name=>'a', :value=>1)
@@ -1586,15 +1561,11 @@ describe "Postgres::Database functions, languages, schemas, and triggers" do
     @d[:test].filter(:name=>'a').all.must_equal [{:name=>'a', :value=>1}]
     @d[:test].filter(:name=>'a').update(:value=>3)
     @d[:test].filter(:name=>'a').all.must_equal [{:name=>'a', :value=>3}]
-    @d.send(:drop_trigger_sql, :test, :identity).must_equal 'DROP TRIGGER identity ON "test"'
     @d.drop_trigger(:test, :identity)
-    @d.send(:create_trigger_sql, :test, :identity, :tf, :after=>true, :events=>:insert, :args=>[1, 'a']).must_equal 'CREATE TRIGGER identity AFTER INSERT ON "test" EXECUTE PROCEDURE tf(1, \'a\')'
-    @d.send(:drop_trigger_sql, :test, :identity, :if_exists=>true, :cascade=>true).must_equal 'DROP TRIGGER IF EXISTS identity ON "test" CASCADE'
     # Make sure if exists works
     @d.drop_trigger(:test, :identity, :if_exists=>true, :cascade=>true)
 
     if @d.supports_trigger_conditions?
-      @d.send(:create_trigger_sql, :test, :identity, :tf, :each_row=>true, :when=> {Sequel[:new][:name] => 'b'}).must_equal %q{CREATE TRIGGER identity BEFORE INSERT OR UPDATE OR DELETE ON "test" FOR EACH ROW WHEN ("new"."name" = 'b') EXECUTE PROCEDURE tf()}
       @d.create_trigger(:test, :identity, :tf, :each_row=>true, :events => :update, :when=> {Sequel[:new][:name] => 'b'})
       @d[:test].filter(:name=>'a').update(:value=>nil)
       @d[:test].filter(:name=>'a').all.must_equal [{:name=>'a', :value=>nil}]