sequel 4.24.0 → 4.25.0

data/lib/sequel/adapters/shared/postgres.rb

@@ -625,6 +625,15 @@ module Sequel
         end
       end
 
+      # Literalize non-String collate options. This is because unquoted collatations
+      # are folded to lowercase, and PostgreSQL used mixed case or capitalized collations.
+      def column_definition_collate_sql(sql, column)
+        if collate = column[:collate]
+          collate = literal(collate) unless collate.is_a?(String)
+          sql << " COLLATE #{collate}"
+        end
+      end
+
       # Handle PostgreSQL specific default format.
       def column_schema_normalize_default(default, type)
         if m = POSTGRES_DEFAULT_RE.match(default)
@@ -1200,7 +1209,7 @@ module Sequel
       LOCK_MODES = ['ACCESS SHARE', 'ROW SHARE', 'ROW EXCLUSIVE', 'SHARE UPDATE EXCLUSIVE', 'SHARE', 'SHARE ROW EXCLUSIVE', 'EXCLUSIVE', 'ACCESS EXCLUSIVE'].each(&:freeze)
 
       Dataset.def_sql_method(self, :delete, [['if server_version >= 90100', %w'with delete from using where returning'], ['else', %w'delete from using where returning']])
-      Dataset.def_sql_method(self, :insert, [['if server_version >= 90100', %w'with insert into columns values returning'], ['else', %w'insert into columns values returning']])
+      Dataset.def_sql_method(self, :insert, [['if server_version >= 90500', %w'with insert into columns values conflict returning'], ['elsif server_version >= 90100', %w'with insert into columns values returning'], ['else', %w'insert into columns values returning']])
       Dataset.def_sql_method(self, :select, [['if opts[:values]', %w'values order limit'], ['elsif server_version >= 80400', %w'with select distinct columns from join where group having window compounds order limit lock'], ['else', %w'select distinct columns from join where group having compounds order limit lock']])
       Dataset.def_sql_method(self, :update, [['if server_version >= 90100', %w'with update table set from where returning'], ['else', %w'update table set from where returning']])
 
@@ -1208,7 +1217,7 @@ module Sequel
       module PreparedStatementMethods
         # Override insert action to use RETURNING if the server supports it.
         def run
-          if @prepared_type == :insert
+          if @prepared_type == :insert && (opts[:returning_pk] || !opts[:returning])
             fetch_rows(prepared_sql){|r| return r.values.first}
           else
             super
@@ -1217,7 +1226,10 @@ module Sequel
 
         def prepared_sql
           return @prepared_sql if @prepared_sql
-          @opts[:returning] = insert_pk if @prepared_type == :insert
+          if @prepared_type == :insert && !opts[:returning]
+            @opts[:returning] = insert_pk
+            @opts[:returning_pk] = true
+          end
           super
           @prepared_sql
         end
@@ -1342,6 +1354,50 @@ module Sequel
         end
       end
 
+      # Handle uniqueness violations when inserting, by updating the conflicting row, using
+      # ON CONFLICT. With no options, uses ON CONFLICT DO NOTHING.  Options:
+      # :constraint :: An explicit constraint name, has precendence over :target.
+      # :target :: The column name or expression to handle uniqueness violations on.
+      # :update :: A hash of columns and values to set.  Uses ON CONFLICT DO UPDATE.
+      # :update_where :: A WHERE condition to use for the update.
+      #
+      # Examples:
+      #
+      #   DB[:table].insert_conflict.insert(:a=>1, :b=>2)
+      #   # INSERT INTO TABLE (a, b) VALUES (1, 2)
+      #   # ON CONFLICT DO NOTHING
+      #   
+      #   DB[:table].insert_conflict(:constraint=>:table_a_uidx).insert(:a=>1, :b=>2)
+      #   # INSERT INTO TABLE (a, b) VALUES (1, 2)
+      #   # ON CONFLICT ON CONSTRAINT table_a_uidx DO NOTHING
+      #   
+      #   DB[:table].insert_conflict(:target=>:a).insert(:a=>1, :b=>2)
+      #   # INSERT INTO TABLE (a, b) VALUES (1, 2)
+      #   # ON CONFLICT (a) DO NOTHING
+      #   
+      #   DB[:table].insert_conflict(:target=>:a, :update=>{:b=>:excluded__b}).insert(:a=>1, :b=>2)
+      #   # INSERT INTO TABLE (a, b) VALUES (1, 2)
+      #   # ON CONFLICT (a) DO UPDATE SET b = excluded.b
+      #   
+      #   DB[:table].insert_conflict(:constraint=>:table_a_uidx,
+      #     :update=>{:b=>:excluded__b}, :update_where=>{:table__status_id=>1}).insert(:a=>1, :b=>2)
+      #   # INSERT INTO TABLE (a, b) VALUES (1, 2)
+      #   # ON CONFLICT ON CONSTRAINT table_a_uidx
+      #   # DO UPDATE SET b = excluded.b WHERE (table.status_id = 1)
+      def insert_conflict(opts=OPTS)
+        clone(:insert_conflict => opts)
+      end
+
+      # Ignore uniqueness/exclusion violations when inserting, using ON CONFLICT DO NOTHING.
+      # Exists mostly for compatibility to MySQL's insert_ignore. Example:
+      #
+      #   DB[:table].insert_ignore.insert(:a=>1, :b=>2)
+      #   # INSERT INTO TABLE (a, b) VALUES (1, 2)
+      #   # ON CONFLICT DO NOTHING
+      def insert_ignore
+        insert_conflict
+      end
+
       # Insert a record returning the record inserted.  Always returns nil without
       # inserting a query if disable_insert_returning is used.
       def insert_select(*values)
@@ -1396,11 +1452,26 @@ module Sequel
         true
       end
 
+      # PostgreSQL 9.5+ supports GROUP CUBE
+      def supports_group_cube?
+        server_version >= 90500
+      end
+
+      # PostgreSQL 9.5+ supports GROUP ROLLUP
+      def supports_group_rollup?
+        server_version >= 90500
+      end
+
       # True unless insert returning has been disabled for this dataset.
       def supports_insert_select?
         !@opts[:disable_insert_returning]
       end
 
+      # PostgreSQL 9.5+ supports the ON CONFLICT clause to INSERT.
+      def supports_insert_conflict?
+        server_version >= 90500
+      end
+
       # PostgreSQL 9.3rc1+ supports lateral subqueries
       def supports_lateral_subqueries?
         server_version >= 90300
@@ -1503,6 +1574,34 @@ module Sequel
         join_from_sql(:USING, sql)
       end
 
+      # Add ON CONFLICT clause if it should be used
+      def insert_conflict_sql(sql)
+        if opts = @opts[:insert_conflict]
+          sql << " ON CONFLICT"
+
+          if target = opts[:constraint] 
+            sql << " ON CONSTRAINT "
+            identifier_append(sql, target)
+          elsif target = opts[:target]
+            sql << ' ('
+            identifier_append(sql, target)
+            sql << ')'
+          end
+
+          if values = opts[:update]
+            sql << " DO UPDATE SET "
+            update_sql_values_hash(sql, values)
+            if where = opts[:update_where]
+              sql << " WHERE "
+              literal_append(sql, where)
+            end
+          else
+            sql << " DO NOTHING"
+          end
+
+        end
+      end
+
       # Return the primary key to use for RETURNING in an INSERT statement
       def insert_pk
         if (f = opts[:from]) && !f.empty?

data/lib/sequel/adapters/shared/sqlite.rb

@@ -335,6 +335,7 @@ module Sequel
         /\ACHECK constraint failed/ => CheckConstraintViolation,
         /\A(SQLITE ERROR 19 \(CONSTRAINT\) : )?constraint failed\z/ => ConstraintViolation,
         /may not be NULL\z|NOT NULL constraint failed: .+\z/ => NotNullConstraintViolation,
+        /\ASQLITE ERROR \d+ \(\) : CHECK constraint failed: / => CheckConstraintViolation
       }.freeze
       def database_error_regexps
         DATABASE_ERROR_REGEXPS

data/lib/sequel/adapters/swift/sqlite.rb

@@ -11,6 +11,18 @@ module Sequel
         extend Sequel::Database::ResetIdentifierMangling
         include Sequel::SQLite::DatabaseMethods
 
+        DATABASE_ERROR_REGEXPS = {
+          /\AUNIQUE constraint failed: / => UniqueConstraintViolation,
+          /\AFOREIGN KEY constraint failed/ => ForeignKeyConstraintViolation,
+          /\ACHECK constraint failed/ => CheckConstraintViolation,
+          /\A(SQLITE ERROR 19 \(CONSTRAINT\) : )?constraint failed/ => ConstraintViolation,
+          /may not be NULL\z|NOT NULL constraint failed: .+/ => NotNullConstraintViolation,
+          /\ASQLITE ERROR \d+ \(\) : CHECK constraint failed: / => CheckConstraintViolation
+        }.freeze
+        def database_error_regexps
+          DATABASE_ERROR_REGEXPS
+        end
+
         # Set the correct pragmas on the connection.
         def connect(opts)
           c = super

data/lib/sequel/database/schema_generator.rb

@@ -81,6 +81,10 @@ module Sequel
       #
       # The following options are supported:
       #
+      # :collate :: The collation to use for the column.  For backwards compatibility,
+      #             only symbols and string values are supported, and they are used verbatim.
+      #             However, on PostgreSQL, symbols are literalized as regular identifiers,
+      #             since unquoted collations are unlikely to be valid.
       # :default :: The default value for the column.
       # :deferrable :: For foreign key columns, this ensures referential integrity will work even if
       #                referencing table uses a foreign key value that does not

data/lib/sequel/database/schema_methods.rb

@@ -524,7 +524,9 @@ module Sequel
 
     # Add collate SQL fragment to column creation SQL.
     def column_definition_collate_sql(sql, column)
-      sql << " COLLATE #{column[:collate]}" if column[:collate]
+      if collate = column[:collate]
+        sql << " COLLATE #{collate}"
+      end
     end
 
     # Add default SQL fragment to column creation SQL.

data/lib/sequel/dataset/actions.rb

@@ -915,7 +915,7 @@ module Sequel
       db = @db
       if db.sharded?
         opts = Hash[opts]
-        opts[:server] = @opts[:server] || :read_only
+        opts[:server] = @opts[:server] || (@opts[:lock] ? :default : :read_only)
         opts
       end
       db.execute(sql, opts, &block)

data/lib/sequel/dataset/prepared_statements.rb

@@ -90,13 +90,19 @@ module Sequel
       # The argument to supply to insert and update, which may use
       # placeholders specified by prepared_args
       attr_accessor :prepared_modify_values
-      
+
       # Sets the prepared_args to the given hash and runs the
       # prepared statement.
       def call(bind_vars={}, &block)
         bind(bind_vars).run(&block)
       end
 
+      # Raise an error if attempting to call prepare on an already
+      # prepared statement.
+      def prepare(*)
+        raise Error, "cannot prepare an already prepared statement"
+      end
+
       # Send the columns to the original dataset, as calling it
       # on the prepared statement can cause problems.
       def columns
@@ -164,12 +170,14 @@ module Sequel
           with_sql(prepared_sql).first
         when :first
           first
-        when :insert
-          insert(*@prepared_modify_values)
-        when :update
-          update(*@prepared_modify_values)
-        when :delete
-          delete
+        when :insert, :update, :delete
+          if opts[:returning] && supports_returning?(@prepared_type)
+            returning_fetch_rows(prepared_sql)
+          elsif @prepared_type == :delete
+            delete
+          else
+            send(@prepared_type, *@prepared_modify_values)
+          end
         when Array
           case @prepared_type.at(0)
           when :map, :to_hash, :to_hash_groups

data/lib/sequel/dataset/query.rb

@@ -34,7 +34,7 @@ module Sequel
     # Methods that return modified datasets
     QUERY_METHODS = (<<-METHS).split.map(&:to_sym) + JOIN_METHODS
       add_graph_aliases and distinct except exclude exclude_having exclude_where
-      filter for_update from from_self graph grep group group_and_count group_by having intersect invert
+      filter for_update from from_self graph grep group group_and_count group_append group_by having intersect invert
       limit lock_style naked offset or order order_append order_by order_more order_prepend qualify
       reverse reverse_order select select_all select_append select_group select_more server
       set_graph_aliases unfiltered ungraphed ungrouped union
@@ -314,6 +314,17 @@ module Sequel
       select_group(*columns, &block).select_more(COUNT_OF_ALL_AS_COUNT)
     end
 
+    # Returns a copy of the dataset with the given columns added to the list of
+    # existing columns to group on. If no existing columns are present this
+    # method simply sets the columns as the initial ones to group on.
+    #
+    #   DB[:items].group_append(:b) # SELECT * FROM items GROUP BY b
+    #   DB[:items].group(:a).group_append(:b) # SELECT * FROM items GROUP BY a, b
+    def group_append(*columns, &block)
+      columns = @opts[:group] + columns if @opts[:group]
+      group(*columns, &block)
+    end
+
     # Adds the appropriate CUBE syntax to GROUP BY.
     def group_cube
       raise Error, "GROUP BY CUBE not supported on #{db.database_type}" unless supports_group_cube?
@@ -576,7 +587,10 @@ module Sequel
     # A symbol may be used for database independent locking behavior, but
     # all supported symbols have separate methods (e.g. for_update).
     #
-    #   DB[:items].lock_style('FOR SHARE NOWAIT') # SELECT * FROM items FOR SHARE NOWAIT
+    #   DB[:items].lock_style('FOR SHARE NOWAIT')
+    #   # SELECT * FROM items FOR SHARE NOWAIT
+    #   DB[:items].lock_style('FOR UPDATE OF table1 SKIP LOCKED')
+    #   # SELECT * FROM items FOR UPDATE OF table1 SKIP LOCKED
     def lock_style(style)
       clone(:lock => style)
     end

data/lib/sequel/dataset/sql.rb

@@ -1010,8 +1010,7 @@ module Sequel
 
     # An expression for how to handle an empty array lookup.
     def empty_array_value(op, cols)
-      c = Array(cols)
-      SQL::BooleanExpression.from_value_pairs(c.zip(c), :AND, op == :IN)
+      {1 => ((op == :IN) ? 0 : 1)}
     end
     
     # Format the timestamp based on the default_timestamp_format, with a couple
@@ -1527,27 +1526,31 @@ module Sequel
     end
 
     def update_set_sql(sql)
-      values = opts[:values]
       sql << SET
+      values = @opts[:values]
       if values.is_a?(Hash)
-        c = false
-        eq = EQUAL
-        values.each do |k, v|
-          sql << COMMA if c
-          if k.is_a?(String) && !k.is_a?(LiteralString)
-            quote_identifier_append(sql, k)
-          else
-            literal_append(sql, k)
-          end
-          sql << eq
-          literal_append(sql, v)
-          c ||= true
-        end
+        update_sql_values_hash(sql, values)
       else
         sql << values
       end
     end
 
+    def update_sql_values_hash(sql, values)
+      c = false
+      eq = EQUAL
+      values.each do |k, v|
+        sql << COMMA if c
+        if k.is_a?(String) && !k.is_a?(LiteralString)
+          quote_identifier_append(sql, k)
+        else
+          literal_append(sql, k)
+        end
+        sql << eq
+        literal_append(sql, v)
+        c ||= true
+      end
+    end
+
     def update_update_sql(sql)
       sql << UPDATE
     end

data/lib/sequel/extensions/empty_array_consider_nulls.rb

@@ -0,0 +1,35 @@
+# This changes Sequel's literalization of IN/NOT IN with an empty
+# array value to consider NULL values if one of the referenced
+# columns is NULL:
+#
+#   DB[:test].where(:name=>[])
+#   # SELECT * FROM test WHERE (name != name)
+#   DB[:test].exclude(:name=>[])
+#   # SELECT * FROM test WHERE (name = name)
+#
+# The default Sequel behavior is to ignore NULLs, as the above
+# query is not generally optimized well by databases.
+#
+# You can load this extension into specific datasets:
+#
+#   ds = DB[:table]
+#   ds = ds.extension(:empty_array_consider_nulls)
+#
+# Or you can load it into all of a database's datasets, which
+# is probably the desired behavior if you are using this extension:
+#
+#   DB.extension(:empty_array_consider_nulls)
+
+#
+module Sequel
+  module EmptyArrayConsiderNulls
+    # Use a simple expression that is always true or false, never NULL.
+    def empty_array_value(op, cols)
+      c = Array(cols)
+      SQL::BooleanExpression.from_value_pairs(c.zip(c), :AND, op == :IN)
+    end
+    
+  end
+
+  Dataset.register_extension(:empty_array_consider_nulls, EmptyArrayConsiderNulls)
+end

data/lib/sequel/extensions/empty_array_ignore_nulls.rb

@@ -1,34 +1,3 @@
-# This changes Sequel's literalization of IN/NOT IN with an empty
-# array value to not return NULL even if one of the referenced
-# columns is NULL:
-#
-#   DB[:test].where(:name=>[])
-#   # SELECT * FROM test WHERE (1 = 0)
-#   DB[:test].exclude(:name=>[])
-#   # SELECT * FROM test WHERE (1 = 1)
-#
-# The default Sequel behavior is to respect NULLs, so that when
-# name is NULL, the expression returns NULL.
-#
-# You can load this extension into specific datasets:
-#
-#   ds = DB[:table]
-#   ds = ds.extension(:empty_array_ignore_nulls)
-#
-# Or you can load it into all of a database's datasets, which
-# is probably the desired behavior if you are using this extension:
-#
-#   DB.extension(:empty_array_ignore_nulls)
-
-#
-module Sequel
-  module EmptyArrayIgnoreNulls
-    # Use a simple expression that is always true or false, never NULL.
-    def empty_array_value(op, cols)
-      {1 => ((op == :IN) ? 0 : 1)}
-    end
-    
-  end
-
-  Dataset.register_extension(:empty_array_ignore_nulls, EmptyArrayIgnoreNulls)
-end
+# This only exists for backwards compatibility, as the behavior
+# added by this extension is now the default Sequel behavior. 
+Sequel::Dataset.register_extension(:empty_array_ignore_nulls){}

data/lib/sequel/extensions/pg_json_ops.rb

@@ -34,7 +34,7 @@
 #   jb = :jsonb_column.pg_jsonb
 #
 # This creates a Sequel::Postgres::JSONOp or Sequel::Postgres::JSONBOp object that can be used
-# for easier querying:
+# for easier querying.  The following methods are available for both JSONOp and JSONBOp instances:
 #
 #   j[1]                     # (json_column -> 1)
 #   j[%w'a b']               # (json_column #> ARRAY['a','b'])
@@ -56,6 +56,14 @@
 #   j.to_record              # json_to_record(json_column)
 #   j.to_recordset           # json_to_recordset(json_column)
 #
+# There are additional methods are are only supported on JSONBOp instances:
+#
+#   j.contain_all(:a)  # (jsonb_column ?& a)
+#   j.contain_any(:a)  # (jsonb_column ?| a)
+#   j.contains(:h)     # (jsonb_column @> h)
+#   j.contained_by(:h) # (jsonb_column <@ h)
+#   j.has_key?('a')    # (jsonb_column ? 'a')
+#
 # If you are also using the pg_json extension, you should load it before
 # loading this extension.  Doing so will allow you to use the #op method on
 # JSONHash, JSONHarray, JSONBHash, and JSONBArray, allowing you to perform json/jsonb operations