sequel 5.9.0 → 5.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49e114f3c425c0cdcd798fdcb170ac2112fada51077235f8e53b3d9a3940954a
-  data.tar.gz: db61fd6ab89b98c48fd2461f500db1f35afd2624808721733a8efbdc4c14a9ba
+  metadata.gz: 629d77185f762e5399741ae9b1caf03c8cd2bbe5f72fd9487900dc0c4bfc2e7a
+  data.tar.gz: ee46b9c3180a67eb356b1f2ae7b274cc293687b333bf216b76a3e55def081c57
 SHA512:
-  metadata.gz: ff2793a7d47c0fe5ff7669ef248cd2c66bd1761f36a9ca0d2742adb8ee59be88a12264df0155864e7ac34f2795ff8f1a633d4e97fc87bcd9bb7976e93b67615b
-  data.tar.gz: a5996916449312e4a83ecc1e4bdaf2a3789c98321fa622389d675b3b6caf5cc84ce9e413198c2d875d0963ac07b7f3b59f2e80cd50b7eb9f45110b1f87263489
+  metadata.gz: 2b5a5c9ed14dcd9bc6c98286bb4c22637052bb2f152f352d5002b096e2d361e2ac6ec15dcd44881e3e96e37b6acd8896161ad94cac4b674d55a6f54a9d43a948
+  data.tar.gz: 847891aeebf175eca649d58461c38d3127311594bb3906bb73429726230b57dd1555ffbd760e3d5f7b30bb9907d5c6d461af1ac0d3b8e166b8dcbf61787fea97

data/CHANGELOG

@@ -1,3 +1,37 @@
+=== 5.10.0 (2018-07-01)
+
+* Use input type casts when using the postgres adapter with pg 0.18+ to reduce string allocations for some primitive types used as prepared statement arguments (jeremyevans)
+
+* Assume local time if database timezone not specified when handling BC timestamps on JRuby 9.2.0.0 in the pg_extended_date_support extension (jeremyevans)
+
+* Fix parsing of timetz types in the jdbc/postgresql adapter (jeremyevans)
+
+* Make SQLTime.parse respect SQLTime.date and Sequel.application_timezone (jeremyevans)
+
+* Add :top as an option in the list plugin (celsworth) (#1526)
+
+* Fix Model#{ancestors,descendants,self_and_siblings} in the tree plugin when custom parent/children association names are used (jeremyevans) (#1525)
+
+* Treat read-only mode error as disconnect error on mysql and mysql2 adapters, for better behavior on AWS Aurora cluster (jeremyevans)
+
+* Don't use cached placeholder literalizers for in Dataset#{first,where_all,where_each,where_single_value} if argument is empty array or hash (jeremyevans)
+
+* Support :tablespace option when adding tables, indexes, and materialized views on PostgreSQL (jeremyevans)
+
+* Support :include option for indexes on PostgreSQL 11+ (jeremyevans)
+
+* Allow the use of IN/NOT IN operators with set returning functions for Sequel::Model datasets (jeremyevans)
+
+* Make many_to_pg_array associations in the pg_array_associations plugin work on PostgreSQL 11 (jeremyevans)
+
+* Only load strscan library in pg_array extension if it is needed (jeremyevans)
+
+* Don't remove related many_to_one associations from cache when setting column value to existing value for model instances that have not been persisted (jeremyevans) (#1521)
+
+* Support ruby 2.6+ endless ranges in the pg_range extension (jeremyevans)
+
+* Support ruby 2.6+ endless ranges in filters, using just a >= operator for them (jeremyevans)
+
 === 5.9.0 (2018-06-01)
 
 * Support generated columns on MySQL 5.7+ and MariaDB 5.2+ (wjordan, jeremyevans) (#1517)

data/doc/release_notes/5.10.0.txt

@@ -0,0 +1,84 @@
+= New Features
+
+* Ruby 2.6+ endless ranges are now supported as condition specifier
+  values, using a >= operator for them:
+
+    DB[:t].where(c: 1...)
+    # SELECT * FROM t WHERE (c >= 1)
+
+* Ruby 2.6+ endless ranges are now supported in the pg_range
+  extension:
+
+    DB[:t].where(id: 1).update(r: 1...)
+    # UPDATE t SET r = '[1,)' WHERE (id = 1)
+
+* The :include option when creating indexes is now supported on
+  PostgreSQL 11, specifying additional columns to include in the index
+  without indexing them.  This is useful to allow index only scans in
+  additional cases.
+
+* The :tablespace option is now supported when creating tables,
+  indexes, and materialized views on PostgreSQL.
+
+* The list plugin now supports a :top option, which can be used to
+  specify the top of the list.  The default value for the top of the
+  list is 1, but using this option you can make the top of the list be
+  0.
+
+= Other Improvements
+
+* In the pg_array_associations plugin, filtering by associations for
+  many_to_pg_array associations now works correctly on PostgreSQL 11.
+  Previously it did not work on PostgreSQL 11 due to new restrictions
+  on using set returning functions in the the SELECT list.
+
+* When setting the value of a column to the same value the column
+  already has, for a new model object that has not yet been persisted,
+  where the column is used as the foreign key for at least one
+  many_to_one association, do not clear any related associations from
+  the associations cache.
+
+* In the pg_array extension, if there are separate conversion procs for
+  timetz and time types, the conversion proc for the timetz[] type now
+  correctly uses the conversion proc for the timetz type to convert
+  scalar values, instead of the conversion proc for the time type.
+
+* Empty arrays and hashes are now correctly handled in
+  Dataset#{first,where_all,where_each,where_single_value} when a
+  cached placeholder literalizer is used.
+
+* In the tree plugin, Model#{ancestors,descendants,self_and_siblings}
+  now work correctly when custom parent/children association names
+  are used.
+
+* The inner loop of the postgres adapter row fetching code is now
+  2-3% faster.
+
+* When using the postgres adapter with pg-0.18+, set a
+  type_map_for_queries for the connection to allow it to handle input
+  type casts for Integer, Float, TrueClass, and FalseClass values
+  without allocating strings.
+
+* SQLTime.parse (and therefore Sequel.string_to_time) now respects the
+  SQLTime.date and Sequel.application_timezone settings.
+
+* The jdbc/postgresql adapter now correctly parses timetz types.
+
+* On JRuby 9.2.0.0, when handling BC timestamps without timezones in
+  the pg_extended_date_support extension, assume local time and not
+  UTC time if the database timezone is not specified and
+  Sequel.datetime_class is Time.
+
+* Errors indicating that a MySQL database is in read-only mode are
+  now treated as disconnect errors in the mysql and mysql2 adapters,
+  for better behavior in failover scenarios.
+
+* Sequel::Model datasets now support the use of IN/NOT IN operators
+  where the second argument for the operator (the right hand side) is
+  a set returning function.  Previously, the Sequel::Model code
+  assumed the right hand side of an IN/NOT IN operator was a datasets
+  or array, since those are the only values where Sequel will
+  automatically create such an operator.
+
+* Sequel no longer loads the strscan library in the pg_array extension
+  if it is not necessary because the parser from sequel_pg is used.

data/lib/sequel/adapters/jdbc/postgresql.rb

@@ -214,7 +214,7 @@ module Sequel
         STRING_TYPE = Java::JavaSQL::Types::VARCHAR
         ARRAY_TYPE = Java::JavaSQL::Types::ARRAY
         ARRAY_METHOD = Postgres.method(:RubyPGArray)
-        PG_SPECIFIC_TYPES = [ARRAY_TYPE, Java::JavaSQL::Types::OTHER, Java::JavaSQL::Types::STRUCT].freeze
+        PG_SPECIFIC_TYPES = [ARRAY_TYPE, Java::JavaSQL::Types::OTHER, Java::JavaSQL::Types::STRUCT, Java::JavaSQL::Types::TIME_WITH_TIMEZONE, Java::JavaSQL::Types::TIME].freeze
         HSTORE_METHOD = Postgres.method(:RubyPGHstore)
 
         def type_convertor(map, meta, type, i)

data/lib/sequel/adapters/postgres.rb

@@ -15,6 +15,12 @@ begin
   end
 
   Sequel::Postgres::USES_PG = true
+  if defined?(PG::TypeMapByClass)
+    type_map = Sequel::Postgres::PG_QUERY_TYPE_MAP = PG::TypeMapByClass.new
+    type_map[Integer] = PG::TextEncoder::Integer.new
+    type_map[FalseClass] = type_map[TrueClass] = PG::TextEncoder::Boolean.new
+    type_map[Float] = PG::TextEncoder::Float.new
+  end
 rescue LoadError => e 
   begin
     require 'postgres-pr/postgres-compat'
@@ -211,6 +217,9 @@ module Sequel
         end
 
         conn.instance_variable_set(:@db, self)
+        if USES_PG && conn.respond_to?(:type_map_for_queries=) && defined?(PG_QUERY_TYPE_MAP)
+          conn.type_map_for_queries = PG_QUERY_TYPE_MAP
+        end
 
         if encoding = opts[:encoding] || opts[:charset]
           if conn.respond_to?(:set_client_encoding)
@@ -737,9 +746,9 @@ module Sequel
         cols = []
         procs = db.conversion_procs
         res.nfields.times do |fieldnum|
-          cols << [fieldnum, procs[res.ftype(fieldnum)], output_identifier(res.fname(fieldnum))]
+          cols << [procs[res.ftype(fieldnum)], output_identifier(res.fname(fieldnum))]
         end
-        self.columns = cols.map{|c| c[2]}
+        self.columns = cols.map{|c| c[1]}
         cols
       end
       
@@ -756,13 +765,20 @@ module Sequel
       # For each row in the result set, yield a hash with column name symbol
       # keys and typecasted values.
       def yield_hash_rows(res, cols)
-        res.ntuples.times do |recnum|
+        ntuples = res.ntuples
+        recnum = 0
+        while recnum < ntuples
+          fieldnum = 0
+          nfields = cols.length
           converted_rec = {}
-          cols.each do |fieldnum, type_proc, fieldsym|
+          while fieldnum < nfields
+            type_proc, fieldsym = cols[fieldnum]
             value = res.getvalue(recnum, fieldnum)
             converted_rec[fieldsym] = (value && type_proc) ? type_proc.call(value) : value
+            fieldnum += 1 
           end
           yield converted_rec
+          recnum += 1
         end
       end
     end

data/lib/sequel/adapters/shared/postgres.rb

@@ -1042,6 +1042,10 @@ module Sequel
           sql += " ON COMMIT #{ON_COMMIT[on_commit]}"
         end
 
+        if tablespace = options[:tablespace]
+          sql += " TABLESPACE #{quote_identifier(tablespace)}"
+        end
+
         if server = options[:foreign]
           sql += " SERVER #{quote_identifier(server)}"
           if foreign_opts = options[:options]
@@ -1077,7 +1081,13 @@ module Sequel
 
       # DDL fragment for initial part of CREATE VIEW statement
       def create_view_prefix_sql(name, options)
-        create_view_sql_append_columns("CREATE #{'OR REPLACE 'if options[:replace]}#{'TEMPORARY 'if options[:temp]}#{'RECURSIVE ' if options[:recursive]}#{'MATERIALIZED ' if options[:materialized]}VIEW #{quote_schema_table(name)}", options[:columns] || options[:recursive])
+        sql = create_view_sql_append_columns("CREATE #{'OR REPLACE 'if options[:replace]}#{'TEMPORARY 'if options[:temp]}#{'RECURSIVE ' if options[:recursive]}#{'MATERIALIZED ' if options[:materialized]}VIEW #{quote_schema_table(name)}", options[:columns] || options[:recursive])
+
+        if tablespace = options[:tablespace]
+          sql += " TABLESPACE #{quote_identifier(tablespace)}"
+        end
+
+        sql
       end
 
       # SQL for dropping a function from the database.
@@ -1147,7 +1157,7 @@ module Sequel
         when :spatial
           index_type = :gist
         end
-        "CREATE #{unique}INDEX#{' CONCURRENTLY' if index[:concurrently]}#{if_not_exists} #{quote_identifier(index_name)} ON #{quote_schema_table(table_name)} #{"USING #{index_type} " if index_type}#{expr}#{filter}"
+        "CREATE #{unique}INDEX#{' CONCURRENTLY' if index[:concurrently]}#{if_not_exists} #{quote_identifier(index_name)} ON #{quote_schema_table(table_name)} #{"USING #{index_type} " if index_type}#{expr}#{" INCLUDE #{literal(Array(index[:include]))}" if index[:include]}#{" TABLESPACE #{quote_identifier(index[:tablespace])}" if index[:tablespace]}#{filter}"
       end
 
       # Setup datastructures shared by all postgres adapters.

data/lib/sequel/adapters/utils/mysql_mysql2.rb

@@ -17,6 +17,7 @@ module Sequel
         MySQL client is not connected
         This connection is still waiting for a result, try again once you have the result
         closed MySQL connection
+        The MySQL server is running with the --read-only option so it cannot execute this statement
         END
         # Error messages for mysql and mysql2 that indicate the current connection should be disconnected
         MYSQL_DATABASE_DISCONNECT_ERRORS = /\A#{Regexp.union(disconnect_errors)}/

data/lib/sequel/database/schema_generator.rb

@@ -222,6 +222,9 @@ module Sequel
       #                  operations on the table while the index is being
       #                  built.
       # :opclass :: Use a specific operator class in the index.
+      # :include :: Include additional column values in the index, without
+      #             actually indexing on those values (PostgreSQL 11+).
+      # :tablespace :: Specify tablespace for index.
       #
       # Microsoft SQL Server specific options:
       #

data/lib/sequel/database/schema_methods.rb

@@ -181,6 +181,7 @@ module Sequel
     #             where keys are option names and values are option values.  Note
     #             that option names are unquoted, so you should not use untrusted
     #             keys.
+    # :tablespace :: The tablespace to use for the table.
     #
     # See <tt>Schema::CreateTableGenerator</tt> and the {"Schema Modification" guide}[rdoc-ref:doc/schema_modification.rdoc].
     def create_table(name, options=OPTS, &block)
@@ -281,6 +282,7 @@ module Sequel
     #               in a subquery, if you are providing a Dataset as the source
     #               argument, if should probably call the union method with the
     #               all: true and from_self: false options.
+    # :tablespace :: The tablespace to use for materialized views.
     def create_view(name, source, options = OPTS)
       execute_ddl(create_view_sql(name, source, options))
       remove_cached_schema(name)

data/lib/sequel/dataset/actions.rb

@@ -231,10 +231,11 @@ module Sequel
 
           return res
         end
+        where_args = args
         args = arg
       end
 
-      if loader = cached_placeholder_literalizer(:_first_cond_loader) do |pl|
+      if loader = cached_where_placeholder_literalizer(where_args||args, block, :_first_cond_loader) do |pl|
           _single_record_ds.where(pl.arg)
         end
 
@@ -875,7 +876,7 @@ module Sequel
     #   DB[:table].where_all(id: [1,2,3])
     #   # SELECT * FROM table WHERE (id IN (1, 2, 3))
     def where_all(cond, &block)
-      if loader = _where_loader
+      if loader = _where_loader([cond], nil)
         loader.all(filter_expr(cond), &block)
       else
         where(cond).all(&block)
@@ -889,7 +890,7 @@ module Sequel
     #   DB[:table].where_each(id: [1,2,3]){|row| p row}
     #   # SELECT * FROM table WHERE (id IN (1, 2, 3))
     def where_each(cond, &block)
-      if loader = _where_loader
+      if loader = _where_loader([cond], nil)
         loader.each(filter_expr(cond), &block)
       else
         where(cond).each(&block)
@@ -904,7 +905,7 @@ module Sequel
     #   DB[:table].select(:name).where_single_value(id: 1)
     #   # SELECT name FROM table WHERE (id = 1) LIMIT 1
     def where_single_value(cond)
-      if loader = cached_placeholder_literalizer(:_where_single_value_loader) do |pl|
+      if loader = cached_where_placeholder_literalizer([cond], nil, :_where_single_value_loader) do |pl|
           single_value_ds.where(pl.arg)
         end
 
@@ -1039,8 +1040,8 @@ module Sequel
     end
 
     # Loader used for where_all and where_each.
-    def _where_loader
-      cached_placeholder_literalizer(:_where_loader) do |pl|
+    def _where_loader(where_args, where_block)
+      cached_where_placeholder_literalizer(where_args, where_block, :_where_loader) do |pl|
         where(pl.arg)
       end
     end

data/lib/sequel/dataset/misc.rb

@@ -309,6 +309,20 @@ module Sequel
       loader
     end
 
+    # Return a cached placeholder literalizer for the key, unless where_block is
+    # nil and where_args is an empty array or hash.  This is designed to guard
+    # against placeholder literalizer use when passing arguments to where
+    # in the uncached case and filter_expr if a cached placeholder literalizer
+    # is used.
+    def cached_where_placeholder_literalizer(where_args, where_block, key, &block)
+      where_args = where_args[0] if where_args.length == 1
+      unless where_block
+        return if where_args == OPTS || where_args == EMPTY_ARRAY
+      end
+
+      cached_placeholder_literalizer(key, &block)
+    end
+
     # Set the columns for the current dataset.
     def columns=(v)
       cache_set(:_columns, v)

data/lib/sequel/extensions/pg_array.rb

@@ -65,12 +65,12 @@
 # If you want an easy way to call PostgreSQL array functions and
 # operators, look into the pg_array_ops extension.
 #
-# This extension requires the strscan and delegate libraries.
+# This extension requires the delegate library, and the strscan library
+# sequel_pg has not been loaded.
 #
 # Related module: Sequel::Postgres::PGArray
 
 require 'delegate'
-require 'strscan'
 
 module Sequel
   module Postgres
@@ -99,7 +99,7 @@ module Sequel
             register_array_type('bytea', :oid=>1001, :scalar_oid=>17, :type_symbol=>:blob)
             register_array_type('date', :oid=>1182, :scalar_oid=>1082)
             register_array_type('time without time zone', :oid=>1183, :scalar_oid=>1083, :type_symbol=>:time)
-            register_array_type('time with time zone', :oid=>1270, :scalar_oid=>1083, :type_symbol=>:time_timezone, :scalar_typecast=>:time)
+            register_array_type('time with time zone', :oid=>1270, :scalar_oid=>1266, :type_symbol=>:time_timezone, :scalar_typecast=>:time)
 
             register_array_type('smallint', :oid=>1005, :scalar_oid=>21, :scalar_typecast=>:integer)
             register_array_type('oid', :oid=>1028, :scalar_oid=>26, :scalar_typecast=>:integer)
@@ -300,93 +300,97 @@ module Sequel
         end
       end
 
-      # PostgreSQL array parser that handles PostgreSQL array output format.
-      # Note that does not handle all forms out input that PostgreSQL will
-      # accept, and it will not raise an error for all forms of invalid input.
-      class Parser < StringScanner
-        # Set the source for the input, and any converter callable
-        # to call with objects to be created.  For nested parsers
-        # the source may contain text after the end current parse,
-        # which will be ignored.
-        def initialize(source, converter=nil)
-          super(source)
-          @converter = converter 
-          @stack = [[]]
-          @encoding = string.encoding
-          @recorded = String.new.force_encoding(@encoding)
-        end
+      unless Sequel::Postgres.respond_to?(:parse_pg_array)
+        require 'strscan'
+
+        # PostgreSQL array parser that handles PostgreSQL array output format.
+        # Note that does not handle all forms out input that PostgreSQL will
+        # accept, and it will not raise an error for all forms of invalid input.
+        class Parser < StringScanner
+          # Set the source for the input, and any converter callable
+          # to call with objects to be created.  For nested parsers
+          # the source may contain text after the end current parse,
+          # which will be ignored.
+          def initialize(source, converter=nil)
+            super(source)
+            @converter = converter 
+            @stack = [[]]
+            @encoding = string.encoding
+            @recorded = String.new.force_encoding(@encoding)
+          end
 
-        # Take the buffer of recorded characters and add it to the array
-        # of entries, and use a new buffer for recorded characters.
-        def new_entry(include_empty=false)
-          if !@recorded.empty? || include_empty
-            entry = @recorded
-            if entry == 'NULL' && !include_empty
-              entry = nil
-            elsif @converter
-              entry = @converter.call(entry)
+          # Take the buffer of recorded characters and add it to the array
+          # of entries, and use a new buffer for recorded characters.
+          def new_entry(include_empty=false)
+            if !@recorded.empty? || include_empty
+              entry = @recorded
+              if entry == 'NULL' && !include_empty
+                entry = nil
+              elsif @converter
+                entry = @converter.call(entry)
+              end
+              @stack.last.push(entry)
+              @recorded = String.new.force_encoding(@encoding)
             end
-            @stack.last.push(entry)
-            @recorded = String.new.force_encoding(@encoding)
           end
-        end
 
-        # Parse the input character by character, returning an array
-        # of parsed (and potentially converted) objects.
-        def parse
-          raise Sequel::Error, "invalid array, empty string" if eos?
-          raise Sequel::Error, "invalid array, doesn't start with {" unless scan(/((\[\d+:\d+\])+=)?\{/)
-
-          while !eos?
-            char = scan(/[{}",]|[^{}",]+/)
-            if char == ','
-              # Comma outside quoted string indicates end of current entry
-              new_entry
-            elsif char == '"'
-              raise Sequel::Error, "invalid array, opening quote with existing recorded data" unless @recorded.empty?
-              while true
-                char = scan(/["\\]|[^"\\]+/)
-                if char == '\\'
-                  @recorded << getch
-                elsif char == '"'
-                  n = peek(1)
-                  raise Sequel::Error, "invalid array, closing quote not followed by comma or closing brace" unless n == ',' || n == '}'
-                  break
+          # Parse the input character by character, returning an array
+          # of parsed (and potentially converted) objects.
+          def parse
+            raise Sequel::Error, "invalid array, empty string" if eos?
+            raise Sequel::Error, "invalid array, doesn't start with {" unless scan(/((\[\d+:\d+\])+=)?\{/)
+
+            while !eos?
+              char = scan(/[{}",]|[^{}",]+/)
+              if char == ','
+                # Comma outside quoted string indicates end of current entry
+                new_entry
+              elsif char == '"'
+                raise Sequel::Error, "invalid array, opening quote with existing recorded data" unless @recorded.empty?
+                while true
+                  char = scan(/["\\]|[^"\\]+/)
+                  if char == '\\'
+                    @recorded << getch
+                  elsif char == '"'
+                    n = peek(1)
+                    raise Sequel::Error, "invalid array, closing quote not followed by comma or closing brace" unless n == ',' || n == '}'
+                    break
+                  else
+                    @recorded << char
+                  end
+                end
+                new_entry(true)
+              elsif char == '{'
+                raise Sequel::Error, "invalid array, opening brace with existing recorded data" unless @recorded.empty?
+
+                # Start of new array, add it to the stack
+                new = []
+                @stack.last << new
+                @stack << new
+              elsif char == '}'
+                # End of current array, add current entry to the current array
+                new_entry
+
+                if @stack.length == 1
+                  raise Sequel::Error, "array parsing finished without parsing entire string" unless eos?
+
+                  # Top level of array, parsing should be over.
+                  # Pop current array off stack and return it as result
+                  return @stack.pop
                 else
-                  @recorded << char
+                  # Nested array, pop current array off stack
+                  @stack.pop
                 end
-              end
-              new_entry(true)
-            elsif char == '{'
-              raise Sequel::Error, "invalid array, opening brace with existing recorded data" unless @recorded.empty?
-
-              # Start of new array, add it to the stack
-              new = []
-              @stack.last << new
-              @stack << new
-            elsif char == '}'
-              # End of current array, add current entry to the current array
-              new_entry
-
-              if @stack.length == 1
-                raise Sequel::Error, "array parsing finished without parsing entire string" unless eos?
-
-                # Top level of array, parsing should be over.
-                # Pop current array off stack and return it as result
-                return @stack.pop
               else
-                # Nested array, pop current array off stack
-                @stack.pop
+                # Add the character to the recorded character buffer.
+                @recorded << char
               end
-            else
-              # Add the character to the recorded character buffer.
-              @recorded << char
             end
-          end
 
-          raise Sequel::Error, "array parsing finished with array unclosed"
+            raise Sequel::Error, "array parsing finished with array unclosed"
+          end
         end
-      end unless Sequel::Postgres.respond_to?(:parse_pg_array)
+      end
 
       # Callable object that takes the input string and parses it using Parser.
       class Creator