sequel 5.6.0 → 5.7.0

data/Rakefile

@@ -49,7 +49,7 @@ task :website_rdoc=>[:website_rdoc_main, :website_rdoc_adapters, :website_rdoc_p
 RDoc::Task.new(:website_rdoc_main) do |rdoc|
   rdoc.rdoc_dir = "www/public/rdoc"
   rdoc.options += RDOC_OPTS + %w'--no-ignore-invalid'
-  rdoc.rdoc_files.add %w"README.rdoc CHANGELOG MIT-LICENSE lib/*.rb lib/sequel/*.rb lib/sequel/{connection_pool,dataset,database,model}/*.rb doc/*.rdoc doc/release_notes/*.txt lib/sequel/extensions/migration.rb"
+  rdoc.rdoc_files.add %w"README.rdoc CHANGELOG doc/CHANGELOG.old MIT-LICENSE lib/*.rb lib/sequel/*.rb lib/sequel/{connection_pool,dataset,database,model}/*.rb doc/*.rdoc doc/release_notes/*.txt lib/sequel/extensions/migration.rb"
 end
 
 RDoc::Task.new(:website_rdoc_adapters) do |rdoc|

data/doc/opening_databases.rdoc

@@ -230,8 +230,6 @@ The following additional options are supported:
 There are a few issues with specific jdbc driver gems:
 
 jdbc-h2 :: jdbc-h2 versions greater than 1.3.175 have issues with ORDER BY not working correctly in some cases.
-jdbc-postgres :: jdbc-postgres 9.4.1204 has issues using PostgreSQL-specific types in bound variables.
-                 Use jdbc-postgres 9.4.1200 or below.
 
 === mysql 
 

data/doc/postgresql.rdoc

@@ -139,6 +139,21 @@ allows you do create an unlogged table by specifying the <tt>unlogged: true</tt>
   DB.create_table(:table, unlogged: true){Integer :i}
   # CREATE UNLOGGED TABLE "table" ("i" integer)
 
+=== Creating Identity Columns
+
+You can use the +:identity+ option when creating columns to mark them as identity columns.
+Identity columns are tied to a sequence for the default value.  You can still override the
+default value for the column when inserting:
+
+  DB.create_table(:table){Integer :id, identity: true}
+  # CREATE TABLE "table" ("id" integer GENERATED BY DEFAULT AS IDENTITY)
+
+If you want to disallow using a user provided value when inserting, you can mark the
+identity column using <tt>identity: :always</tt>:
+
+  DB.create_table(:table){Integer :id, identity: :always}
+  # CREATE TABLE "table" ("id" integer GENERATED ALWAYS AS IDENTITY)
+
 === Creating/Dropping Schemas, Languages, Functions, and Triggers
 
 Sequel has built in support for creating and dropping PostgreSQL schemas, procedural languages, functions, and triggers:
@@ -255,6 +270,22 @@ conditions, the constraint violation will be ignored, but the row will not be up
   # ON CONFLICT ON CONSTRAINT table_a_uidx
   # DO UPDATE SET b = excluded.b WHERE (table.status_id = 1)
 
+=== INSERT OVERRIDING SYSTEM|USER VALUE Support
+
+PostgreSQL 10+ supports identity columns, which are designed to replace the serial
+columns previously used for autoincrementing primary keys.  You can use
+Dataset#overriding_system_value and Dataset#overriding_user_value to use this new
+syntax:
+
+  DB.create_table(:table){primary_key :id}
+  # Ignore the given value for id, using the identity's sequence value
+  DB[:table].overriding_user_value.insert(:id=>1)
+
+  DB.create_table(:table){primary_key :id, :identity=>:always}
+  # Force the use of the given value for id, because otherwise the insert will
+  # raise an error, since GENERATED ALWAYS was using when creating the column.
+  DB[:table].overriding_system_value.insert(:id=>1)
+
 === Distinct On Specific Columns
 
 Sequel allows passing columns to <tt>Dataset#distinct</tt>, which will make the dataset return

data/doc/querying.rdoc

@@ -111,8 +111,8 @@ you want:
 If you want the value for multiple columns, you can pass an array to
 <tt>Sequel::Dataset#get</tt>:
 
-  artist_id, artist_name,  = Artist.get([:id, :name])
-  # SELECT name FROM artists LIMIT 1
+  artist_id, artist_name = Artist.get([:id, :name])
+  # SELECT id, name FROM artists LIMIT 1
   # => [1, "YJM"]
 
 === Retrieving Multiple Objects

data/doc/release_notes/5.7.0.txt

@@ -0,0 +1,108 @@
+= New Features
+
+* An integer64 extension has been added, which treats the Integer
+  class as a generic 64-bit integer type.  Sequel's default behavior
+  for Integer is to use the integer type, which on most databases
+  is a 32-bit type.
+
+  This affects all internal use of the Integer class as a generic
+  database type, so that methods like primary_key and foreign_key
+  also default to using a 64-bit integer type when using this
+  extension.
+
+* When using PostgreSQL 10+, you can use the :identity option when
+  creating columns to create identity columns:
+
+    DB.create_table(:table){Integer :id, identity: true}
+    # CREATE TABLE "table" ("id" integer GENERATED BY DEFAULT AS IDENTITY)
+
+  If you want to disallow using a user provided value when inserting,
+  or updating you can use a value of :always:
+
+    DB.create_table(:table){Integer :id, identity: :always}
+    # CREATE TABLE "table" ("id" integer GENERATED ALWAYS AS IDENTITY)
+
+* Database#convert_serial_to_identity has been added on PostgreSQL 10.2+.
+  This method can convert existing serial columns to identity columns
+  in most cases, but it currently requires superuser permissions as it
+  modifies the system tables directly.
+
+* Dataset#overriding_system_value and #overriding_user_value are
+  now supported on PostgreSQL to work with identity columns.  You can
+  use #overriding_system_value to force the use of a user provided
+  value for identity columns that are GENERATED ALWAYS, and you can
+  use #overriding_user_value to ignore any user value for identity
+  columns and always use the next entry in the sequence.
+
+= Other Improvements
+
+* On PostgreSQL 10.2+, identity columns are now used instead of serial
+  columns as the default for auto incrementing primary keys:
+
+    DB.create_table(:table){primary_key :id}
+
+    # Sequel 5.7.0+ and PostgreSQL 10.2+
+    # CREATE TABLE "table" ("id" integer
+    #   GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)
+
+    # Older Sequel version or older PostgreSQL version
+    # CREATE TABLE "table" ("id" serial PRIMARY KEY)
+
+  Identity columns fix many issues that serial columns have, in
+  addition to being the SQL standard way to support auto incrementing
+  columns.
+
+* PostgreSQL identity columns are now correctly recognized and the
+  :auto_increment schema entry is now populated for them.
+
+* Dataset#with_sql_{all,each,first,single_value} now use a cached
+  dataset to avoid clobbering the current dataset's columns.
+  Previously, the clobbering of the current dataset's columns was
+  documented and the method warned against using SQL with
+  different columns.  These methods are now safe to use in such
+  cases, but will not have the same performance advantages if the
+  current dataset is not cached.
+
+* On ruby 2.1+, Sequel now uses Process::CLOCK_MONOTONIC when
+  performing elapsed time calculations so that it is not affected by
+  modifications to the system's time.
+
+* In the postgres adapter, prepared statement errors related to
+  changing types are now treated as disconnect errors. While they
+  are not technically disconnect errors, treating them as such
+  will in general reduce the total number of exceptions generated
+  from 1 per affected statement per connection to 1 per
+  connection.
+
+* In the pg_array_associations plugin, the array_type for
+  pg_array_to_many and many_to_pg_array association reflections is
+  now always the scalar type for the array (e.g. integer). Previously,
+  the array type (e.g. integer[]) was used in some cases.  This didn't
+  previously result in issues as PostgreSQL considers integer[][] the
+  same type as integer[].
+
+* In the pg_array_associations plugin, the many_to_pg_array
+  association remove_all_* method now uses the appropriate cast to
+  work for non-integer array types such as bigint[].
+
+* Database#server_version on PostgreSQL 10.1+ now works correctly
+  when the connection does not support the server_version method.
+  Now the server_version_num database setting is always used to
+  ensure consistent behavior across adapters.
+
+* In the jdbc/oracle adapter, temporary clobs are now manually
+  freed to prevent a memory leak, in line with the Oracle JDBC
+  driver recommendations.
+
+* The Sequel <4 release notes and changelog are no longer shipped
+  with the gem, decreasing the size of the gem by 20%.
+
+= Backwards Compatibility
+
+* The switch to using identity columns instead of serial columns
+  by default on PostgreSQL 10.2+ may break backwards compatibilty
+  in some situations, such as code that relies on what are generally
+  considered bugs in serial columns, such as CREATE TABLE LIKE
+  using the same sequence for the column in both the existing table
+  and the new table, or that dropping the default value for the
+  column does not drop the related sequence.

data/doc/testing.rdoc

@@ -159,6 +159,7 @@ SEQUEL_ERROR_SQL :: Use the error_sql extension when running the specs
 SEQUEL_INDEX_CACHING :: Use the index_caching extension when running the specs
 SEQUEL_FREEZE_DATABASE :: Freeze the database before running the integration specs
 SEQUEL_IDENTIFIER_MANGLING :: Use the identifier_mangling extension when running the specs
+SEQUEL_INTEGER64 :: Use the integer64 extension when running the adapter or integration specs
 SEQUEL_MODEL_PREPARED_STATEMENTS :: Use the prepared_statements plugin when running the specs
 SEQUEL_NO_CACHE_ASSOCIATIONS :: Don't cache association metadata when running the specs
 SEQUEL_NO_CHECK_SQLS :: Don't check for specific SQL syntax when running the specs

data/lib/sequel/adapters/jdbc/derby.rb

@@ -36,7 +36,7 @@ module Sequel
 
         # Derby uses an IDENTITY sequence for autoincrementing columns.
         def serial_primary_key_options
-          {:primary_key => true, :type => :integer, :identity=>true, :start_with=>1}
+          {:primary_key => true, :type => Integer, :identity=>true, :start_with=>1}
         end
 
         # The SVN version of the database.

data/lib/sequel/adapters/jdbc/oracle.rb

@@ -28,6 +28,13 @@ module Sequel
         end
       end 
 
+      def self.OracleClob(r, i)
+        return unless clob = r.getClob(i)
+        str = clob.getSubString(1, clob.length)
+        clob.freeTemporary if clob.isTemporary
+        str
+      end
+
       module DatabaseMethods
         include Sequel::Oracle::DatabaseMethods
         include Sequel::JDBC::Transactions
@@ -104,6 +111,7 @@ module Sequel
         def setup_type_convertor_map
           super
           @type_convertor_map[:OracleDecimal] = Oracle.method(:OracleDecimal)
+          @type_convertor_map[:OracleClob] = Oracle.method(:OracleClob)
         end
       end
       
@@ -112,6 +120,7 @@ module Sequel
 
         NUMERIC_TYPE = Java::JavaSQL::Types::NUMERIC
         TIMESTAMP_TYPE = Java::JavaSQL::Types::TIMESTAMP
+        CLOB_TYPE = Java::JavaSQL::Types::CLOB
         TIMESTAMPTZ_TYPES = [Java::oracle.jdbc.OracleTypes::TIMESTAMPTZ, Java::oracle.jdbc.OracleTypes::TIMESTAMPLTZ].freeze
 
         def type_convertor(map, meta, type, i)
@@ -124,6 +133,8 @@ module Sequel
             end
           when *TIMESTAMPTZ_TYPES
             map[TIMESTAMP_TYPE]
+          when CLOB_TYPE 
+            map[:OracleClob]
           else
             super
           end

data/lib/sequel/adapters/postgres.rb

@@ -45,6 +45,7 @@ module Sequel
       DISCONNECT_ERROR_CLASSES.freeze
       
       disconnect_errors = [
+        'ERROR:  cached plan must not change result type',
         'could not receive data from server',
         'no connection to the server',
         'connection not open',

data/lib/sequel/adapters/shared/postgres.rb

@@ -213,6 +213,76 @@ module Sequel
         run("COMMIT PREPARED #{literal(transaction_id)}", opts)
       end
 
+      # Convert the first primary key column in the +table+ from being a serial column to being an identity column.
+      # If the column is already an identity column, assume it was already converted and make no changes.
+      #
+      # Only supported on PostgreSQL 10.2+, since on those versions Sequel will use identity columns
+      # instead of serial columns for auto incrementing primary keys. Only supported when running as
+      # a superuser, since regular users cannot modify system tables, and there is no way to keep an
+      # existing sequence when changing an existing column to be an identity column.
+      #
+      # This method can raise an exception in at least the following cases where it may otherwise succeed
+      # (there may be additional cases not listed here):
+      #
+      # * The serial column was added after table creation using PostgreSQL <7.3
+      # * A regular index also exists on the column (such an index can probably be dropped as the
+      #   primary key index should suffice)
+      #
+      # Options:
+      # :column :: Specify the column to convert instead of using the first primary key column
+      # :server :: Run the SQL on the given server
+      def convert_serial_to_identity(table, opts=OPTS)
+        raise Error, "convert_serial_to_identity is only supported on PostgreSQL 10.2+" unless server_version >= 100002
+
+        server = opts[:server]
+        server_hash = server ? {:server=>server} : {}
+        ds = dataset
+        ds = ds.server(server) if server
+
+        raise Error, "convert_serial_to_identity requires superuser permissions" unless ds.get{current_setting('is_superuser')} == 'on'
+
+        table_oid = regclass_oid(table)
+        im = input_identifier_meth
+        unless column = im.call(opts[:column] || ((sch = schema(table).find{|col, sch| sch[:primary_key] && sch[:auto_increment]}) && sch[0]))
+          raise Error, "could not determine column to convert from serial to identity automatically"
+        end
+
+        column_num = ds.from(:pg_attribute).
+          where(:attrelid=>table_oid, :attname=>column).
+          get(:attnum)
+
+        pg_class = Sequel.cast('pg_class', :regclass)
+        res = ds.from(:pg_depend).
+          where(:refclassid=>pg_class, :refobjid=>table_oid, :refobjsubid=>column_num, :classid=>pg_class, :objsubid=>0, :deptype=>%w'a i').
+          select_map([:objid, Sequel.as({:deptype=>'i'}, :v)])
+
+        case res.length
+        when 0
+          raise Error, "unable to find related sequence when converting serial to identity"
+        when 1
+          seq_oid, already_identity = res.first
+        else
+          raise Error, "more than one linked sequence found when converting serial to identity"
+        end
+
+        return if already_identity
+
+        transaction(server_hash) do
+          run("ALTER TABLE #{quote_schema_table(table)} ALTER COLUMN #{quote_identifier(column)} DROP DEFAULT", server_hash)
+
+          ds.from(:pg_depend).
+            where(:classid=>pg_class, :objid=>seq_oid, :objsubid=>0, :deptype=>'a').
+            update(:deptype=>'i')
+
+          ds.from(:pg_attribute).
+            where(:attrelid=>table_oid, :attname=>column).
+            update(:attidentity=>'d')
+        end
+
+        remove_cached_schema(table)
+        nil
+      end
+
       # Creates the function in the database.  Arguments:
       # name :: name of the function to create
       # definition :: string definition of the function, or object file for a dynamically loaded C function.
@@ -528,23 +598,16 @@ module Sequel
       # PostgreSQL uses SERIAL psuedo-type instead of AUTOINCREMENT for
       # managing incrementing primary keys.
       def serial_primary_key_options
-        {:primary_key => true, :serial => true, :type=>Integer}
+        auto_increment_key = server_version >= 100002 ? :identity : :serial
+        {:primary_key => true, auto_increment_key => true, :type=>Integer}
       end
 
       # The version of the PostgreSQL server, used for determining capability.
       def server_version(server=nil)
         return @server_version if @server_version
-        @server_version = synchronize(server) do |conn|
-          (conn.server_version rescue nil) if conn.respond_to?(:server_version)
-        end
-        unless @server_version
-          @server_version = if m = /PostgreSQL (\d+)\.(\d+)(?:(?:rc\d+)|\.(\d+))?/.match(fetch('SELECT version()').single_value)
-            (m[1].to_i * 10000) + (m[2].to_i * 100) + m[3].to_i
-          else
-            0
-          end
-        end
-        @server_version
+        ds = dataset
+        ds = ds.server(server) if server
+        @server_version ||= ds.with_sql("SELECT CAST(current_setting('server_version_num') AS integer) AS v").single_value rescue 0
       end
 
       # PostgreSQL supports CREATE TABLE IF NOT EXISTS on 9.1+
@@ -709,6 +772,17 @@ module Sequel
         end
       end
 
+      # Support identity columns, but only use the identity SQL syntax if no
+      # default value is given.
+      def column_definition_default_sql(sql, column)
+        super
+        if !column[:default] && (identity = column[:identity])
+          sql << " GENERATED "
+          sql << (identity == :always ? "ALWAYS" : "BY DEFAULT")
+          sql << " AS IDENTITY"
+        end
+      end
+
       # Handle PostgreSQL specific default format.
       def column_schema_normalize_default(default, type)
         if m = /\A(?:B?('.*')::[^']+|\((-?\d+(?:\.\d+)?)\))\z/.match(default)
@@ -1115,6 +1189,11 @@ module Sequel
           where{pg_attribute[:attnum] > 0}.
           where{{pg_class[:oid]=>oid}}.
           order{pg_attribute[:attnum]}
+
+        if server_version > 100000
+          ds = ds.select_append{pg_attribute[:attidentity]}
+        end
+
         ds.map do |row|
           row[:default] = nil if blank_object?(row[:default])
           if row[:base_oid]
@@ -1127,8 +1206,9 @@ module Sequel
             row.delete(:db_base_type)
           end
           row[:type] = schema_column_type(row[:db_type])
+          identity = row.delete(:attidentity)
           if row[:primary_key]
-            row[:auto_increment] = !!(row[:default] =~ /\Anextval/io)
+            row[:auto_increment] = !!(row[:default] =~ /\A(?:nextval)/i) || identity == 'a' || identity == 'd'
           end
           [m.call(row.delete(:name)), row]
         end
@@ -1430,6 +1510,19 @@ module Sequel
         nil
       end
 
+      # Use OVERRIDING USER VALUE for INSERT statements, so that identity columns
+      # always use the user supplied value, and an error is not raised for identity
+      # columns that are GENERATED ALWAYS.
+      def overriding_system_value
+        clone(:override=>:system)
+      end
+
+      # Use OVERRIDING USER VALUE for INSERT statements, so that identity columns
+      # always use the sequence value instead of the user supplied value.
+      def overriding_user_value
+        clone(:override=>:user)
+      end
+
       def supports_cte?(type=:select)
         if type == :select
           server_version >= 80400
@@ -1626,6 +1719,17 @@ module Sequel
         end
       end
 
+      # Support OVERRIDING SYSTEM|USER VALUE in insert statements
+      def insert_values_sql(sql)
+        case opts[:override]
+        when :system
+          sql << " OVERRIDING SYSTEM VALUE"
+        when :user
+          sql << " OVERRIDING USER VALUE"
+        end
+        super
+      end
+
       # For multiple table support, PostgreSQL requires at least
       # two from tables, with joins allowed.
       def join_from_sql(type, sql)

data/lib/sequel/connection_pool/sharded_threaded.rb

@@ -185,24 +185,25 @@ class Sequel::ShardedThreadedConnectionPool < Sequel::ThreadedConnectionPool
       return conn
     end
 
-    time = Time.now
+    timeout = @timeout
+    timer = Sequel.start_timer
 
     sync do
-      @waiters[server].wait(@mutex, @timeout)
+      @waiters[server].wait(@mutex, timeout)
       if conn = next_available(server)
         return(allocated(server)[thread] = conn)
       end
     end
 
     until conn = assign_connection(thread, server)
-      deadline ||= time + @timeout
-      current_time = Time.now
-      raise_pool_timeout(current_time - time, server) if current_time > deadline
+      elapsed = Sequel.elapsed_seconds_since(timer)
+      raise_pool_timeout(elapsed, server) if elapsed > timeout
+
       # :nocov:
       # It's difficult to get to this point, it can only happen if there is a race condition
       # where a connection cannot be acquired even after the thread is signalled by the condition
       sync do
-        @waiters[server].wait(@mutex, deadline - current_time)
+        @waiters[server].wait(@mutex, timeout - elapsed)
         if conn = next_available(server)
           return(allocated(server)[thread] = conn)
         end