sequel 5.74.0 → 5.76.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16c90ef17199e4e48f39edee069981ba0030cf63dc481b136f637b1fe069743e
-  data.tar.gz: 52063f32827a04c33173867207290da294878d898d25b80c37666cf66a54b780
+  metadata.gz: ca7f95e54ed437de173bbdfac08f4091f0e8debfb1a89bb168cd6157341afa9d
+  data.tar.gz: f49e07aec7ae48c509abf3965bc10ec102616d5ccc99ecb49d0116a9162760af
 SHA512:
-  metadata.gz: d808534a13ae702a884524ceae4adcdb8eff9d46681a62d5b2c0d926b46279743350520248311c41f8a385c7fc66d17cbc44c7f1af52f1cffd356fca498d6bb2
-  data.tar.gz: 309d0f0c0a7c47a4f1acc90107443e7fd2aa51cb09038082354e822711b11986ae271bb784e710336cebafc83b8b5a99adf088bfd19e2353728d25de86f942c4
+  metadata.gz: 9b8453bf85f48038160481a3b795a692a670b3e73084e6f531f353739c2549f809af5d45368db4dbcc0ac0bbd318eb4a94a02d0289d8c09117641f0ea4dc493d
+  data.tar.gz: a7daa11f1dc3d757f1618f4e8ee832c33e7072f375cfeea028e68b340414a159c92d05d76d75fd9bbb35cb498699ed9db32c9e7cbb158bb2b8a88e598ade34e6

data/CHANGELOG

@@ -1,3 +1,43 @@
+=== 5.76.0 (2024-01-01)
+
+* Improve performance and flexibility of regexp matching in sqlite adapter (paddor) (#2108)
+
+* Support SQL::Identifier for Database#tables :schema option values on PostgreSQL (jeremyevans)
+
+* Support generating rcte queries using UNION or UNION ALL in the rcte plugin (jonathanfrias) (#2107)
+
+* Make Database#table_exists? on PostgreSQL handle lock or statement timeout errors as evidence the table exists (jeremyevans) (#2106)
+
+* Work around DateTime.jd fractional second bug on JRuby in named_timezones extension (jeremyevans)
+
+* Support fractional times and timestamps on SQLAnywhere (jeremyevans)
+
+* Make round_timestamps extension use Dataset#sqltime_precision for rounding Sequel::SQLTime values (jeremyevans)
+
+* Remove special handling of %N modifier in Dataset#default_timestamp_format (jeremyevans)
+
+* Add Dataset#default_time_format private method, for adapters to override for time (not timestamp) formatting (jeremyevans)
+
+* Remove Dataset#format_timestamp_offset private method (jeremyevans)
+
+* Remove special handling of %z modifier in Dataset#default_timestamp_format (jeremyevans)
+
+* Add Dataset#literal_date_or_time, for simpler use by bound argument code (jeremyevans)
+
+* Add auto_cast_date_and_time extension, for casting date and time values using SQL standard functions (jeremyevans)
+
+=== 5.75.0 (2023-12-01)
+
+* Make any_not_empty? extension support passing pattern argument to any? (jeremyevans) (#2100)
+
+* Respect :skip_transaction option in PostgreSQL Dataset#paged_each (jeremyevans) (#2097)
+
+* Add TimestampMigrator.run_single to run a single migration file up or down (opya, jeremyevans) (#2093)
+
+* Support INSERT RETURNING on MariaDB 10.5+, and use it when saving new model objects (jeremyevans)
+
+* Add Database#{defer,immediate}_constraints on PostgreSQL for changing handling of deferrable constraints in a transaction (jeremyevans)
+
 === 5.74.0 (2023-11-01)
 
 * Make generated columns show up in Database#schema when using SQLite 3.37+ (jeremyevans) (#2087)

data/doc/opening_databases.rdoc

@@ -388,8 +388,10 @@ The following additional options are supported:
 :readonly :: open database in read-only mode
 :timeout :: the busy timeout to use in milliseconds (default: 5000).
 :setup_regexp_function :: Whether to setup a REGEXP function in the underlying SQLite3::Database object. Doing so
-                          allows you to use regexp support in dataset expressions.  Note that this creates a new
-                          Regexp object per call to the function, so it is not an efficient implementation.
+                          allows you to use regexp support in dataset expressions.  If +:cached+ or <tt>"cached"</tt>+, caches each
+                          unique regex (more efficient but risk of memory leak). If a Proc is provided, it will be called with
+                          a string for the regexp and a string for the value to compare, and should return whether the regexp
+                          string matches the string value to compare. The default Proc used does <tt>Regexp.new(regexp_str).match(str)</tt>.
 
 Note that SQLite memory databases are restricted to a single connection by
 default.  This is because SQLite does not allow multiple connections to

data/doc/release_notes/5.75.0.txt

@@ -0,0 +1,35 @@
+= New Features
+
+* Database#{defer,immediate}_constraints methods have been added on
+  PostgreSQL for changing handling of deferrable constraints inside
+  a transaction. defer_constraints sets deferrable constraints to
+  be deferred (not checked until transaction commit), and
+  immediate_constraints sets deferrable constraints to be checked
+  as part of the related query, and any already deferred constraint
+  checks to be applied immediately. You can pass the :constraints
+  option to only apply the changes to specific constraints.
+
+* TimestampMigrator.run_single has been added, to migrate a single
+  migration up or down.
+
+= Other Improvements
+
+* INSERT RETURNING is now supported on MariaDB 10.5+, and used
+  automatically when saving new model objects. Note that this
+  is not supported when using the jdbc adapter, because the
+  jdbc-mysql driver doesn't support it.  A jdbc/mariadb adapter
+  could be added, as it's likely recent versions of the
+  jdbc-mariadb driver would support it, but the jdbc-mariadb gem 
+  hasn't been updated in over 4 years.  Talk to the jdbc-mariadb
+  gem maintainers if you want to use this feature with the jdbc
+  adapter.
+
+* The Dataset#paged_each optimization in the postgres adapter
+  now respects the :skip_transaction option, making it the
+  same as the :hold option.  Note that this has effects beyond
+  just skipping the transaction, but non-HOLD cursors are only
+  supported inside transactions.
+
+* The any_not_empty? extension's Dataset#any? method now supports
+  an argument, passing it to Enumerable#any? (which has supported
+  an argument since Ruby 2.5).

data/doc/release_notes/5.76.0.txt

@@ -0,0 +1,86 @@
+= New Features
+
+* An auto_cast_date_and_time extension has been added, which will
+  automatically cast date and time values using SQL standard functions.
+  This makes sure the database will treat the value as a date, time,
+  or timestamp, instead of treating it as a string or unknown type:
+
+    DB.get(Date.today).class
+    # SELECT '2024-01-01' AS v LIMIT 1
+    String
+
+    DB.extension(:auto_cast_date_and_time)
+    DB.get(Date.today).class
+    # SELECT DATE '2024-01-01' AS v LIMIT 1
+    Date
+
+  This was already Sequel's default behavior on adapters that required
+  it.  This extension is usable on PostgreSQL and MySQL.  It is not
+  usable on SQLite (no date/time types) or Microsoft SQL Server (no
+  support for the SQL standard conversion syntax).
+
+  This extension can break code that currently works. If using it on
+  PostgreSQL, it will cast the values to TIMESTAMP, not TIMESTAMP
+  WITH TIME ZONE, which can break code that depended on an implicit
+  conversion to TIMESTAMP WITH TIME ZONE.  The pg_timestamptz
+  extension integrates with the the auto_cast_date_and_time extension
+  and will implicitly cast Time/DateTime to TIMESTAMP WITH TIME ZONE.
+
+* The sqlite adapter now supports a :cached value for the
+  :setup_regexp_function Database option, which will cache regexp
+  values instead of creating a new regexp per value to compare.  This
+  is much faster when using a regexp comparison on a large dataset,
+  but can result in a memory leak if using dynamic regexps. You can
+  also provide a Proc value for the :setup_regexp_function option,
+  which will be passed both the regexp source string and the database
+  string to compare, and should return whether the database string
+  matches the regexp string.
+
+* The rcte_tree plugin now supports a :union_all option, which can
+  be set to false to use UNION instead of UNION ALL in the recursive
+  common table expression.
+
+= Other Improvements
+
+* Time/DateTime/SQLite literalization speed has more than doubled
+  compared to the previous version.  The internal code is also much
+  simpler, as the speedup resulted from removing multiple abstraction
+  layers that mostly existed for Ruby 1.8 support.
+
+* Database#table_exists? on PostgreSQL now handles lock or statement
+  timeout errors as evidence the table exists.
+
+* The round_timestamps extension now correctly rounds SQLTime values
+  on Microsoft SQL Server (the only database Sequel supports where
+  time precision is different than timestamp precision).
+
+* Fractional times and timestamps are now supported on SQLAnywhere,
+  except for time values when using the jdbc adapter due to a
+  limitation in the JDBC sqlanywhere driver.
+
+* Database#tables and #views on PostgreSQL now supports
+  SQL::Identifier values for the :schema option.
+
+* The named_timezones extension now works around a bug in DateTime.jd
+  on JRuby.
+
+= Backwards Compatibility
+
+* Time/DateTime/SQLTime literalization internals have changed.
+  If you are using an external adapter and the external adapter
+  overrides or calls any of the following methods:
+
+  * requires_sql_standard_datetimes?
+  * supports_timestamp_usecs?
+  * supports_timestamp_timezones?
+  * timestamp_precision
+  * sqltime_precision
+
+  then the adapter may need to be updated to support Sequel 5.76.0.
+  Additionally, if the adapter uses %N or %z in
+  default_timestamp_format, it may need to be updated. Adapters
+  should now just override default_timestamp_format and/or
+  default_time_format methods as appropriate for the database.
+
+* The Dataset#format_timestamp_offset private method has been
+  removed.

data/doc/testing.rdoc

@@ -159,6 +159,7 @@ The SEQUEL_INTEGRATION_URL environment variable specifies the Database connectio
 
 === Other
 
+SEQUEL_AUTO_CAST_DATE_TIME :: Use the auto_cast_date_and_time extension when running the specs
 SEQUEL_ASYNC_THREAD_POOL :: Use the async_thread_pool extension when running the specs
 SEQUEL_ASYNC_THREAD_POOL_PREEMPT :: Use the async_thread_pool extension when running the specs, with the :preempt_async_thread option
 SEQUEL_CHECK_PENDING :: Try running all specs (note, can cause lockups for some adapters), and raise errors for skipped specs that don't fail

data/lib/sequel/adapters/jdbc/h2.rb

@@ -1,6 +1,7 @@
 # frozen-string-literal: true
 
 Sequel::JDBC.load_driver('org.h2.Driver', :H2)
+require_relative '../../extensions/auto_cast_date_and_time'
 
 module Sequel
   module JDBC
@@ -14,6 +15,8 @@ module Sequel
 
     module H2
       module DatabaseMethods
+        include AutoCastDateAndTime
+
         def commit_prepared_transaction(transaction_id, opts=OPTS)
           run("COMMIT TRANSACTION #{transaction_id}", opts)
         end

data/lib/sequel/adapters/jdbc/hsqldb.rb

@@ -2,6 +2,7 @@
 
 Sequel::JDBC.load_driver('org.hsqldb.jdbcDriver', :HSQLDB)
 require_relative 'transactions'
+require_relative '../../extensions/auto_cast_date_and_time'
 
 module Sequel
   module JDBC
@@ -15,6 +16,7 @@ module Sequel
 
     module HSQLDB
       module DatabaseMethods
+        include AutoCastDateAndTime
         include ::Sequel::JDBC::Transactions
 
         def database_type

data/lib/sequel/adapters/jdbc/sqlanywhere.rb

@@ -56,6 +56,17 @@ module Sequel
 
         private
 
+        # JDBC SQLAnywhere driver does not appear to handle fractional
+        # times correctly.
+        def default_time_format
+          "'%H:%M:%S'"
+        end
+
+        # Set to zero to work around JDBC SQLAnywhere driver bug.
+        def sqltime_precision
+          0
+        end
+
         SMALLINT_TYPE = Java::JavaSQL::Types::SMALLINT
         BOOLEAN_METHOD = Object.new
         def BOOLEAN_METHOD.call(r, i)

data/lib/sequel/adapters/mysql2.rb

@@ -182,8 +182,8 @@ module Sequel
             1
           when false
             0
-          when DateTime, Time
-            literal(arg)[1...-1]
+          when Time, Date
+            @default_dataset.literal_date_or_time(arg, true)
           else
             arg
           end

data/lib/sequel/adapters/odbc/mssql.rb

@@ -43,7 +43,7 @@ module Sequel
         # Use ODBC format, not Microsoft format, as the ODBC layer does
         # some translation, but allow for millisecond precision.
         def default_timestamp_format
-          "{ts '%Y-%m-%d %H:%M:%S%N'}"
+          "{ts '%Y-%m-%d %H:%M:%S.%3N'}"
         end
 
         # Use ODBC format, not Microsoft format, as the ODBC layer does

data/lib/sequel/adapters/postgres.rb

@@ -188,8 +188,8 @@ module Sequel
         # :nocov:
         # Not covered by tests as tests use pg_extended_date_support
         # extension, which has basically the same code.
-        when DateTime, Time
-          literal(arg)
+        when Time, DateTime
+          @default_dataset.literal_date_or_time(arg)
         # :nocov:
         else
           arg
@@ -672,6 +672,7 @@ module Sequel
       #          cursor usage.
       # :rows_per_fetch :: The number of rows per fetch (default 1000).  Higher
       #                    numbers result in fewer queries but greater memory use.
+      # :skip_transaction :: Same as :hold, but :hold takes priority.
       #
       # Usage:
       #
@@ -764,13 +765,13 @@ module Sequel
 
       # Use a cursor to fetch groups of records at a time, yielding them to the block.
       def cursor_fetch_rows(sql)
-        server_opts = {:server=>@opts[:server] || :read_only}
         cursor = @opts[:cursor]
-        hold = cursor[:hold]
+        hold = cursor.fetch(:hold){cursor[:skip_transaction]}
+        server_opts = {:server=>@opts[:server] || :read_only, :skip_transaction=>hold}
         cursor_name = quote_identifier(cursor[:cursor_name] || 'sequel_cursor')
         rows_per_fetch = cursor[:rows_per_fetch].to_i
 
-        db.public_send(*(hold ? [:synchronize, server_opts[:server]] : [:transaction, server_opts])) do 
+        db.transaction(server_opts) do 
           begin
             execute_ddl("DECLARE #{cursor_name} NO SCROLL CURSOR WITH#{'OUT' unless hold} HOLD FOR #{sql}", server_opts)
             rows_per_fetch = 1000 if rows_per_fetch <= 0

data/lib/sequel/adapters/shared/mssql.rb

@@ -932,7 +932,7 @@ module Sequel
       # since that is the format that is multilanguage and not
       # DATEFORMAT dependent.
       def default_timestamp_format
-        "'%Y-%m-%dT%H:%M:%S%N%z'"
+        "'%Y-%m-%dT%H:%M:%S.%3N'"
       end
 
       # Only include the primary table in the main delete clause

data/lib/sequel/adapters/shared/mysql.rb

@@ -646,7 +646,7 @@ module Sequel
       MATCH_AGAINST_BOOLEAN = ["MATCH ".freeze, " AGAINST (".freeze, " IN BOOLEAN MODE)".freeze].freeze
 
       Dataset.def_sql_method(self, :delete, %w'with delete from where order limit')
-      Dataset.def_sql_method(self, :insert, %w'insert ignore into columns values on_duplicate_key_update')
+      Dataset.def_sql_method(self, :insert, %w'insert ignore into columns values on_duplicate_key_update returning')
       Dataset.def_sql_method(self, :select, %w'with select distinct calc_found_rows columns from join where group having window compounds order limit lock')
       Dataset.def_sql_method(self, :update, %w'with update ignore table set where order limit')
 
@@ -774,6 +774,21 @@ module Sequel
         clone(:insert_ignore=>true)
       end
 
+      # Support insert select for associations, so that the model code can use
+      # returning instead of a separate query.
+      def insert_select(*values)
+        return unless supports_insert_select?
+        # Handle case where query does not return a row
+        server?(:default).with_sql_first(insert_select_sql(*values)) || false
+      end
+
+      # The SQL to use for an insert_select, adds a RETURNING clause to the insert
+      # unless the RETURNING clause is already present.
+      def insert_select_sql(*values)
+        ds = opts[:returning] ? self : returning
+        ds.insert_sql(*values)
+      end
+
       # Sets up the insert methods to use ON DUPLICATE KEY UPDATE
       # If you pass no arguments, ALL fields will be
       # updated with the new values.  If you pass the fields you
@@ -871,6 +886,11 @@ module Sequel
         true
       end
 
+      # MariaDB 10.5.0 supports INSERT RETURNING.
+      def supports_returning?(type)
+        (type == :insert && db.mariadb? && db.adapter_scheme != :jdbc) ? (db.server_version >= 100500) : false
+      end
+
       # MySQL 8+ supports SKIP LOCKED.
       def supports_skip_locked?
         !db.mariadb? && db.server_version >= 80000
@@ -909,6 +929,16 @@ module Sequel
         super if type == :truncate || @opts[:offset]
       end
 
+      # The strftime format to use when literalizing time (Sequel::SQLTime) values.
+      def default_time_format
+        db.supports_timestamp_usecs? ? super : "'%H:%M:%S'"
+      end
+
+      # The strftime format to use when literalizing timestamp (Time/DateTime) values.
+      def default_timestamp_format
+        db.supports_timestamp_usecs? ? super : "'%Y-%m-%d %H:%M:%S'"
+      end
+
       # Consider the first table in the joined dataset is the table to delete
       # from, but include the others for the purposes of selecting rows.
       def delete_from_sql(sql)

data/lib/sequel/adapters/shared/oracle.rb

@@ -1,6 +1,7 @@
 # frozen-string-literal: true
 
 require_relative '../utils/emulate_offset_with_row_number'
+require_relative '../../extensions/auto_cast_date_and_time'
 
 module Sequel
   module Oracle
@@ -326,6 +327,8 @@ module Sequel
     end
 
     module DatasetMethods
+      include AutoCastDateAndTime
+
       ROW_NUMBER_EXPRESSION = LiteralString.new('ROWNUM').freeze
       BITAND_PROC = lambda{|a, b| Sequel.lit(["CAST(BITAND(", ", ", ") AS INTEGER)"], a, b)}
 
@@ -623,7 +626,7 @@ module Sequel
 
       # The strftime format to use when literalizing the time.
       def default_timestamp_format
-        "TIMESTAMP '%Y-%m-%d %H:%M:%S%N %z'"
+        "'%Y-%m-%d %H:%M:%S.%6N %:z'"
       end
 
       def empty_from_sql
@@ -660,11 +663,6 @@ module Sequel
         super
       end
 
-      # Use a colon for the timestamp offset, since Oracle appears to require it.
-      def format_timestamp_offset(hour, minute)
-        sprintf("%+03i:%02i", hour, minute)
-      end
-
       # Oracle doesn't support empty values when inserting.
       def insert_supports_empty_values?
         false

data/lib/sequel/adapters/shared/postgres.rb

@@ -498,6 +498,25 @@ module Sequel
         :postgres
       end
 
+      # For constraints that are deferrable, defer constraints until 
+      # transaction commit. Options:
+      #
+      # :constraints :: An identifier of the constraint, or an array of
+      #                 identifiers for constraints, to apply this
+      #                 change to specific constraints.
+      # :server :: The server/shard on which to run the query.
+      #
+      # Examples:
+      #
+      #   DB.defer_constraints
+      #   # SET CONSTRAINTS ALL DEFERRED
+      #
+      #   DB.defer_constraints(constraints: [:c1, Sequel[:sc][:c2]])
+      #   # SET CONSTRAINTS "c1", "sc"."s2" DEFERRED
+      def defer_constraints(opts=OPTS)
+        _set_constraints(' DEFERRED', opts)
+      end
+
       # Use PostgreSQL's DO syntax to execute an anonymous code block.  The code should
       # be the literal code string to use in the underlying procedural language.  Options:
       #
@@ -611,6 +630,24 @@ module Sequel
         super
       end
 
+      # Immediately apply deferrable constraints.
+      #
+      # :constraints :: An identifier of the constraint, or an array of
+      #                 identifiers for constraints, to apply this
+      #                 change to specific constraints.
+      # :server :: The server/shard on which to run the query.
+      #
+      # Examples:
+      #
+      #   DB.immediate_constraints
+      #   # SET CONSTRAINTS ALL IMMEDIATE
+      #
+      #   DB.immediate_constraints(constraints: [:c1, Sequel[:sc][:c2]])
+      #   # SET CONSTRAINTS "c1", "sc"."s2" IMMEDIATE
+      def immediate_constraints(opts=OPTS)
+        _set_constraints(' IMMEDIATE', opts)
+      end
+
       # Use the pg_* system tables to determine indexes on a table
       def indexes(table, opts=OPTS)
         m = output_identifier_meth
@@ -1038,6 +1075,31 @@ module Sequel
         end
       end
 
+      # Internals of defer_constraints/immediate_constraints
+      def _set_constraints(type, opts)
+        execute_ddl(_set_constraints_sql(type, opts), opts)
+      end
+
+      # SQL to use for SET CONSTRAINTS
+      def _set_constraints_sql(type, opts)
+        sql = String.new
+        sql << "SET CONSTRAINTS "
+        if constraints = opts[:constraints]
+          dataset.send(:source_list_append, sql, Array(constraints))
+        else
+          sql << "ALL"
+        end
+        sql << type
+      end
+
+      # Consider lock or statement timeout errors as evidence that the table exists
+      # but is locked.
+      def _table_exists?(ds)
+        super
+      rescue DatabaseError => e    
+        raise e unless /canceling statement due to (?:statement|lock) timeout/ =~ e.message 
+      end
+    
       def alter_table_add_column_sql(table, op)
         "ADD COLUMN#{' IF NOT EXISTS' if op[:if_not_exists]} #{column_definition_sql(op)}"
       end
@@ -1451,7 +1513,11 @@ module Sequel
       # currently visible schemas.
       def filter_schema(ds, opts)
         expr = if schema = opts[:schema]
-          schema.to_s
+          if schema.is_a?(SQL::Identifier)
+            schema.value.to_s
+          else
+            schema.to_s
+          end
         else
           Sequel.function(:any, Sequel.function(:current_schemas, false))
         end
@@ -2084,10 +2150,14 @@ module Sequel
         server_version >= 90500
       end
 
+      # :nocov:
+
       # PostgreSQL supports timezones in literal timestamps
       def supports_timestamp_timezones?
+        # SEQUEL6: Remove
         true
       end
+      # :nocov:
 
       # PostgreSQL 8.4+ supports WINDOW clause.
       def supports_window_clause?
@@ -2207,6 +2277,11 @@ module Sequel
         raise(InvalidOperation, "Joined datasets cannot be truncated") if opts[:join]
       end
 
+      # The strftime format to use when literalizing the time.
+      def default_timestamp_format
+        "'%Y-%m-%d %H:%M:%S.%6N%z'"
+      end
+
       # Only include the primary table in the main delete clause
       def delete_from_sql(sql)
         sql << ' FROM '

data/lib/sequel/adapters/shared/sqlanywhere.rb

@@ -281,10 +281,6 @@ module Sequel
         false
       end
 
-      def supports_timestamp_usecs?
-        false
-      end
-
       def supports_window_clause?
         true
       end
@@ -378,6 +374,16 @@ module Sequel
 
       private
 
+      # SQLAnywhere only supports 3 digits after the decimal point for times.
+      def default_time_format
+        "'%H:%M:%S.%3N'"
+      end
+
+      # SQLAnywhere only supports 3 digits after the decimal point for timestamps.
+      def default_timestamp_format
+        "'%Y-%m-%d %H:%M:%S.%3N'"
+      end
+
       # Use 1 for true on Sybase
       def literal_true
         '1'

data/lib/sequel/adapters/shared/sqlite.rb

@@ -917,6 +917,11 @@ module Sequel
         500
       end
 
+      # The strftime format to use when literalizing the time.
+      def default_timestamp_format
+        db.use_timestamp_timezones? ? "'%Y-%m-%d %H:%M:%S.%6N%z'" : super
+      end
+
       # SQL fragment specifying a list of identifiers
       def identifier_list(columns)
         columns.map{|i| quote_identifier(i)}.join(', ')

data/lib/sequel/adapters/sqlite.rb

@@ -111,6 +111,13 @@ module Sequel
       #              static data that you do not want to modify
       # :timeout :: how long to wait for the database to be available if it
       #             is locked, given in milliseconds (default is 5000)
+      # :setup_regexp_function :: enable use of Regexp objects with SQL
+      #             'REGEXP' operator. If the value is :cached or "cached",
+      #             caches the generated regexps, which can result in a memory
+      #             leak if dynamic regexps are used.  If the value is a Proc,
+      #             it will be called with a string for the regexp and a string
+      #             for the value to compare, and should return whether the regexp
+      #             matches.
       def connect(server)
         opts = server_opts(server)
         opts[:database] = ':memory:' if blank_object?(opts[:database])
@@ -126,9 +133,7 @@ module Sequel
         connection_pragmas.each{|s| log_connection_yield(s, db){db.execute_batch(s)}}
 
         if typecast_value_boolean(opts[:setup_regexp_function])
-          db.create_function("regexp", 2) do |func, regexp_str, string|
-            func.result = Regexp.new(regexp_str).match(string) ? 1 : 0
-          end
+          setup_regexp_function(db, opts[:setup_regexp_function])
         end
         
         class << db
@@ -202,6 +207,22 @@ module Sequel
         @conversion_procs['datetime'] = @conversion_procs['timestamp'] = method(:to_application_timestamp)
         set_integer_booleans
       end
+
+      def setup_regexp_function(db, how)
+        case how
+        when Proc
+          # nothing
+        when :cached, "cached"
+          cache = Hash.new{|h,k| h[k] = Regexp.new(k)}
+          how = lambda{|regexp_str, str| cache[regexp_str].match(str)}
+        else
+          how = lambda{|regexp_str, str| Regexp.new(regexp_str).match(str)}
+        end
+
+        db.create_function("regexp", 2) do |func, regexp_str, str|
+          func.result = how.call(regexp_str, str) ? 1 : 0
+        end
+      end
       
       # Yield an available connection.  Rescue
       # any SQLite3::Exceptions and turn them into DatabaseErrors.

data/lib/sequel/dataset/features.rb

@@ -25,11 +25,16 @@ module Sequel
       false
     end
 
+    # :nocov:
+
     # Whether the dataset requires SQL standard datetimes. False by default,
-    # as most allow strings with ISO 8601 format.
+    # as most allow strings with ISO 8601 format. Only for backwards compatibility,
+    # no longer used internally, do not use in new code.
     def requires_sql_standard_datetimes?
+      # SEQUEL6: Remove
       false
     end
+    # :nocov:
 
     # Whether type specifiers are required for prepared statement/bound
     # variable argument placeholders (i.e. :bv__integer), false by default.
@@ -183,10 +188,14 @@ module Sequel
       true
     end
 
+    # :nocov:
+
     # Whether the dataset supports timezones in literal timestamps, false by default.
     def supports_timestamp_timezones?
+      # SEQUEL6: Remove
       false
     end
+    # :nocov:
     
     # Whether the dataset supports fractional seconds in literal timestamps, true by default.
     def supports_timestamp_usecs?

data/lib/sequel/dataset/sql.rb

@@ -82,7 +82,7 @@ module Sequel
       when DateTime
         literal_datetime_append(sql, v)
       when Date
-        sql << literal_date(v)
+        literal_date_append(sql, v)
       when Dataset
         literal_dataset_append(sql, v)
       else
@@ -115,6 +115,33 @@ module Sequel
       sql
     end
 
+    # Literalize a date or time value, as a SQL string value with no
+    # typecasting. If +raw+ is true, remove the surrounding single
+    # quotes.  This is designed for usage by bound argument code that
+    # can work even if the auto_cast_date_and_time extension is
+    # used (either manually or implicitly in the related adapter).
+    def literal_date_or_time(dt, raw=false)
+      value = case dt
+      when SQLTime
+        literal_sqltime(dt)
+      when Time
+        literal_time(dt)
+      when DateTime
+        literal_datetime(dt)
+      when Date
+        literal_date(dt)
+      else
+        raise TypeError, "unsupported type: #{dt.inspect}"
+      end
+
+      if raw
+        value.sub!(/\A'/, '')
+        value.sub!(/'\z/, '')
+      end
+
+      value
+    end
+
     # Returns an array of insert statements for inserting multiple records.
     # This method is used by +multi_insert+ to format insert statements and
     # expects a keys array and and an array of value arrays.
@@ -1104,9 +1131,14 @@ module Sequel
       :"t#{number}"
     end
     
-    # The strftime format to use when literalizing the time.
+    # The strftime format to use when literalizing time (Sequel::SQLTime) values.
+    def default_time_format
+      "'%H:%M:%S.%6N'"
+    end
+
+    # The strftime format to use when literalizing timestamp (Time/DateTime) values.
     def default_timestamp_format
-      requires_sql_standard_datetimes? ? "TIMESTAMP '%Y-%m-%d %H:%M:%S%N%z'" : "'%Y-%m-%d %H:%M:%S%N%z'"
+      "'%Y-%m-%d %H:%M:%S.%6N'"
     end
 
     def delete_delete_sql(sql)
@@ -1169,43 +1201,23 @@ module Sequel
       {1 => ((op == :IN) ? 0 : 1)}
     end
     
-    # Format the timestamp based on the default_timestamp_format, with a couple
-    # of modifiers.  First, allow %N to be used for fractions seconds (if the
-    # database supports them), and override %z to always use a numeric offset
-    # of hours and minutes.
+    # Format the timestamp based on the default_timestamp_format.
     def format_timestamp(v)
-      v2 = db.from_application_timestamp(v)
-      fmt = default_timestamp_format.gsub(/%[Nz]/) do |m|
-        if m == '%N'
-          # Ruby 1.9 supports %N in timestamp formats, but Sequel has supported %N
-          # for longer in a different way, where the . is already appended and only 6
-          # decimal places are used by default.
-          format_timestamp_usec(v.is_a?(DateTime) ? v.sec_fraction*(1000000) : v.usec) if supports_timestamp_usecs?
-        else
-          if supports_timestamp_timezones?
-            # Would like to just use %z format, but it doesn't appear to work on Windows
-            # Instead, the offset fragment is constructed manually
-            minutes = (v2.is_a?(DateTime) ? v2.offset * 1440 : v2.utc_offset/60).to_i
-            format_timestamp_offset(*minutes.divmod(60))
-          end
-        end
-      end
-      v2.strftime(fmt)
+      db.from_application_timestamp(v).strftime(default_timestamp_format)
     end
     
-    # Return the SQL timestamp fragment to use for the timezone offset.
-    def format_timestamp_offset(hour, minute)
-      sprintf("%+03i%02i", hour, minute)
-    end
+    # :nocov:
 
     # Return the SQL timestamp fragment to use for the fractional time part.
     # Should start with the decimal point.  Uses 6 decimal places by default.
     def format_timestamp_usec(usec, ts=timestamp_precision)
+      # SEQUEL6: Remove
       unless ts == 6
         usec = usec/(10 ** (6 - ts))
       end
       sprintf(".%0#{ts}d", usec)
     end
+    # :nocov:
 
     # Append literalization of identifier to SQL string, considering regular strings
     # as SQL identifiers instead of SQL strings.
@@ -1347,11 +1359,12 @@ module Sequel
 
     # SQL fragment for Date, using the ISO8601 format.
     def literal_date(v)
-      if requires_sql_standard_datetimes?
-        v.strftime("DATE '%Y-%m-%d'")
-      else
-        v.strftime("'%Y-%m-%d'")
-      end
+      v.strftime("'%Y-%m-%d'")
+    end
+
+    # Append literalization of date to SQL string.
+    def literal_date_append(sql, v)
+      sql << literal_date(v)
     end
 
     # SQL fragment for DateTime
@@ -1414,7 +1427,7 @@ module Sequel
 
     # SQL fragment for Sequel::SQLTime, containing just the time part
     def literal_sqltime(v)
-      v.strftime("'%H:%M:%S#{format_timestamp_usec(v.usec, sqltime_precision) if supports_timestamp_usecs?}'")
+      v.strftime(default_time_format)
     end
 
     # Append literalization of Sequel::SQLTime to SQL string.

data/lib/sequel/extensions/any_not_empty.rb

@@ -32,8 +32,8 @@
 module Sequel
   module AnyNotEmpty
     # If a block is not given, return whether the dataset is not empty.
-    def any?
-      if defined?(yield)
+    def any?(*a)
+      if !a.empty? || defined?(yield)
         super
       else
         !empty?

data/lib/sequel/extensions/auto_cast_date_and_time.rb

@@ -0,0 +1,94 @@
+# frozen-string-literal: true
+#
+# The auto_cast_date_and_time extension uses SQL standard type casting
+# when literalizing date, time, and timestamp values:
+#
+#   DB.literal(Time.now)
+#   # => "TIMESTAMP '...'"
+#
+#   DB.literal(Date.today)
+#   # => "DATE '...'"
+#
+#   DB.literal(Sequel::SQLTime.create(10, 20, 30))
+#   # => "TIME '10:20:30.000000'"
+#
+# The default behavior of Sequel on adapters that do not require the
+# SQL standard behavior is to format the date or time value without:
+# casting
+#
+#   DB.literal(Sequel::SQLTime.create(10, 20, 30))
+#   # => "'10:20:30.000000'"
+#
+# However, then the database cannot determine the type of the string,
+# and must perform some implicit casting.  If implicit casting cannot
+# be used, it will probably treat the value as a string:
+#
+#  DB.get(Time.now).class
+#  # Without auto_cast_date_and_time: String
+#  #    With auto_cast_date_and_time: Time
+#
+# Note that not all databases support this extension. PostgreSQL and
+# MySQL support it, but SQLite and Microsoft SQL Server do not.
+#
+# You can load this extension into specific datasets:
+#
+#   ds = DB[:table]
+#   ds = ds.extension(:auto_cast_date_and_time)
+#
+# Or you can load it into all of a database's datasets, which
+# is probably the desired behavior if you are using this extension:
+#
+#   DB.extension(:auto_cast_date_and_time)
+# 
+# Related module: Sequel::AutoCastDateAndTime
+
+#
+module Sequel
+  module AutoCastDateAndTime
+    # :nocov:
+
+    # Mark the datasets as requiring sql standard date times.  This is only needed
+    # for backwards compatibility.  
+    def requires_sql_standard_datetimes?
+      # SEQUEL6: Remove
+      true
+    end
+    # :nocov:
+
+    private
+
+    # Explicitly cast SQLTime objects to TIME.
+    def literal_sqltime_append(sql, v)
+      sql << "TIME "
+      super
+    end
+
+    # Explicitly cast Time objects to TIMESTAMP.
+    def literal_time_append(sql, v)
+      sql << literal_datetime_timestamp_cast
+      super
+    end
+
+    # Explicitly cast DateTime objects to TIMESTAMP.
+    def literal_datetime_append(sql, v)
+      sql << literal_datetime_timestamp_cast
+      super
+    end
+
+    # Explicitly cast Date objects to DATE.
+    def literal_date_append(sql, v)
+      sql << "DATE "
+      super
+    end
+
+    # The default cast string to use for Time/DateTime objects.
+    # Respects existing method if already defined.
+    def literal_datetime_timestamp_cast
+      return super if defined?(super)
+      'TIMESTAMP '
+    end
+  end
+
+  Dataset.register_extension(:auto_cast_date_and_time, AutoCastDateAndTime)
+end
+

data/lib/sequel/extensions/migration.rb

@@ -693,6 +693,13 @@ module Sequel
       @migration_tuples = get_migration_tuples
     end
 
+    # Apply the migration in the given file path.  See Migrator.run for the
+    # available options.  Additionally, this method supports the :direction
+    # option for whether to run the migration up (default) or down.
+    def self.run_single(db, path, opts=OPTS)
+      new(db, File.dirname(path), opts).run_single(path, opts[:direction] || :up)
+    end
+
     # The timestamp migrator is current if there are no migrations to apply
     # in either direction.
     def is_current?
@@ -702,20 +709,39 @@ module Sequel
     # Apply all migration tuples on the database
     def run
       migration_tuples.each do |m, f, direction|
-        t = Time.now
-        db.log_info("Begin applying migration #{f}, direction: #{direction}")
-        checked_transaction(m) do
-          m.apply(db, direction)
-          fi = f.downcase
-          direction == :up ? ds.insert(column=>fi) : ds.where(column=>fi).delete
-        end
-        db.log_info("Finished applying migration #{f}, direction: #{direction}, took #{sprintf('%0.6f', Time.now - t)} seconds")
+        apply_migration(m, f, direction)
       end
       nil
     end
 
+    # Apply single migration tuple at the given path with the given direction
+    # on the database.
+    def run_single(path, direction)
+      migration = load_migration_file(path)
+      file_name = File.basename(path)
+      already_applied = applied_migrations.include?(file_name.downcase)
+
+      return if direction == :up ? already_applied : !already_applied
+
+      apply_migration(migration, file_name, direction)
+      nil
+    end
+
     private
 
+    # Apply a single migration with the given filename in the given direction.
+    def apply_migration(migration, file_name, direction)
+      fi = file_name.downcase
+      t = Time.now
+
+      db.log_info("Begin applying migration #{file_name}, direction: #{direction}")
+      checked_transaction(migration) do
+        migration.apply(db, direction)
+        direction == :up ? ds.insert(column=>fi) : ds.where(column=>fi).delete
+      end
+      db.log_info("Finished applying migration #{file_name}, direction: #{direction}, took #{sprintf('%0.6f', Time.now - t)} seconds")
+    end
+
     # Convert the schema_info table to the new schema_migrations table format,
     # using the version of the schema_info table and the current migration files.
     def convert_from_schema_info

data/lib/sequel/extensions/named_timezones.rb

@@ -136,7 +136,7 @@ module Sequel
         v = output_timezone.utc_to_local(v.new_offset(0))
 
         # Force DateTime output instead of TZInfo::DateTimeWithOffset
-        DateTime.jd(v.jd, v.hour, v.minute, v.second + v.sec_fraction, v.offset, v.start)
+        DateTime.civil(v.year, v.month, v.day, v.hour, v.minute, v.second + v.sec_fraction, v.offset, v.start)
       end
       # :nodoc:
       # :nocov:

data/lib/sequel/extensions/pg_array.rb

@@ -241,6 +241,8 @@ module Sequel
             else
               literal(a)
             end
+          when Time, Date
+            @default_dataset.literal_date_or_time(a)
           else
             if (s = bound_variable_arg(a, nil)).is_a?(String)
               bound_variable_array_string(s)

data/lib/sequel/extensions/pg_extended_date_support.rb

@@ -53,8 +53,8 @@ module Sequel
       # on jdbc.
       def bound_variable_arg(arg, conn)
         case arg
-        when Date, Time
-          literal(arg)
+        when Time, Date
+          @default_dataset.literal_date_or_time(arg)
         else
           super
         end
@@ -203,7 +203,7 @@ module Sequel
             date <<= ((date.year) * 24 - 12)
             date = db.from_application_timestamp(date)
             minutes = (date.offset * 1440).to_i
-            date.strftime("'%Y-%m-%d %H:%M:%S.%N#{format_timestamp_offset(*minutes.divmod(60))} BC'")
+            date.strftime("'%Y-%m-%d %H:%M:%S.%6N#{sprintf("%+03i%02i", *minutes.divmod(60))} BC'")
           else
             super
           end
@@ -247,7 +247,7 @@ module Sequel
           def literal_time(time)
             if time < TIME_YEAR_1
               time = db.from_application_timestamp(time)
-              time.strftime("'#{sprintf('%04i', time.year.abs+1)}-%m-%d %H:%M:%S.%N#{format_timestamp_offset(*(time.utc_offset/RATIONAL_60).divmod(60))} BC'")
+              time.strftime("'#{sprintf('%04i', time.year.abs+1)}-%m-%d %H:%M:%S.%6N#{sprintf("%+03i%02i", *(time.utc_offset/RATIONAL_60).divmod(60))} BC'")
             else
               super
             end

data/lib/sequel/extensions/pg_range.rb

@@ -494,8 +494,8 @@ module Sequel
         case k
         when nil
           ''
-        when Date, Time
-          ds.literal(k)[1...-1]
+        when Time, Date
+          ds.literal_date_or_time(k, true)
         when Integer, Float
           k.to_s
         when BigDecimal

data/lib/sequel/extensions/pg_timestamptz.rb

@@ -1,20 +1,35 @@
 # frozen-string-literal: true
 #
 # The pg_timestamptz extension changes the default timestamp
-# type for the database to be +timestamptz+ (+timestamp with time zone+)
-# instead of +timestamp+ (+timestamp without time zone+).  This is
+# type for the database to be +timestamptz+ (<tt>timestamp with time zone</tt>)
+# instead of +timestamp+ (<tt>timestamp without time zone</tt>).  This is
 # recommended if you are dealing with multiple timezones in your application.
+#
+# If you are using the auto_cast_date_and_time extension, the pg_timestamptz
+# extension will automatically cast Time and DateTime values to
+# <tt>TIMESTAMP WITH TIME ZONE</tt> instead of +TIMESTAMP+.
 # 
 # To load the extension into the database:
 #
 #   DB.extension :pg_timestamptz
 #
-# Related module: Sequel::Postgres::Timestamptz
+# To load the extension into individual datasets:
+#
+#   ds = ds.extension(:pg_timestamptz)
+#
+# Note that the loading into individual datasets only affects the integration
+# with the auto_cast_date_and_time extension.
+#
+# Related modules: Sequel::Postgres::Timestamptz, Sequel::Postgres::TimestamptzDatasetMethods
 
 #
 module Sequel
   module Postgres
     module Timestamptz
+      def self.extended(db)
+        db.extend_datasets(TimestamptzDatasetMethods)
+      end
+
       private
 
       # Use timestamptz by default for generic timestamp value.
@@ -22,7 +37,16 @@ module Sequel
         :timestamptz
       end
     end
+
+    module TimestamptzDatasetMethods
+      private
+
+      def literal_datetime_timestamp_cast
+        'TIMESTAMP WITH TIME ZONE '
+      end
+    end
   end
 
+  Dataset.register_extension(:pg_timestamptz, Postgres::TimestamptzDatasetMethods)
   Database.register_extension(:pg_timestamptz, Postgres::Timestamptz)
 end

data/lib/sequel/extensions/round_timestamps.rb

@@ -35,7 +35,7 @@ module Sequel
 
       # Round Sequel::SQLTime values before literalizing
       def literal_sqltime(v)
-        super(v.round(timestamp_precision))
+        super(v.round(sqltime_precision))
       end
 
       # Round Time values before literalizing

data/lib/sequel/plugins/column_encryption.rb

@@ -325,7 +325,7 @@ module Sequel
     #   DB.alter_table(:ce_test) do
     #     c = Sequel[:encrypted_column_name]
     #     add_constraint(:enc_base64) do
-    #       octet_length(decode(regexp_replace(regexp_replace(c, '_', '/', 'g'), '-', '+', 'g'), 'base64')) >= 65}
+    #       octet_length(decode(regexp_replace(regexp_replace(c, '_', '/', 'g'), '-', '+', 'g'), 'base64')) >= 65
     #     end
     #   end
     #

data/lib/sequel/plugins/rcte_tree.rb

@@ -71,6 +71,8 @@ module Sequel
     #              (default: :t)
     # :level_alias :: The symbol identifier to use when eagerly loading descendants
     #                 up to a given level (default: :x_level_x)
+    # :union_all :: Whether to use UNION ALL or UNION with the recursive
+    #               common table expression (default: true)
     module RcteTree
       # Create the appropriate parent, children, ancestors, and descendants
       # associations for the model.
@@ -80,6 +82,7 @@ module Sequel
         opts = opts.dup
         opts[:class] = model
         opts[:methods_module] = Module.new
+        opts[:union_all] = opts[:union_all].nil? ? true : opts[:union_all]
         model.send(:include, opts[:methods_module])
         
         key = opts[:key] ||= :parent_id
@@ -142,7 +145,7 @@ module Sequel
           model.from(SQL::AliasedExpression.new(t, table_alias)).
            with_recursive(t, col_aliases ? base_ds.select(*col_aliases) : base_ds.select_all,
             recursive_ds.select(*c_all),
-            :args=>col_aliases)
+            :args=>col_aliases, union_all: opts[:union_all])
         end
         aal = Array(a[:after_load])
         aal << proc do |m, ancs|
@@ -191,7 +194,7 @@ module Sequel
           table_alias = model.dataset.schema_and_table(model.table_name)[1].to_sym
           ds = model.from(SQL::AliasedExpression.new(t, table_alias)).
             with_recursive(t, base_case, recursive_case,
-             :args=>((key_aliases + col_aliases) if col_aliases))
+             :args=>((key_aliases + col_aliases) if col_aliases), union_all: opts[:union_all])
           ds = r.apply_eager_dataset_changes(ds)
           ds = ds.select_append(ka) unless ds.opts[:select] == nil
           model.eager_load_results(r, eo.merge(:loader=>false, :initialize_rows=>false, :dataset=>ds, :id_map=>nil)) do |obj|
@@ -240,7 +243,7 @@ module Sequel
           model.from(SQL::AliasedExpression.new(t, table_alias)).
            with_recursive(t, col_aliases ? base_ds.select(*col_aliases) : base_ds.select_all,
             recursive_ds.select(*c_all),
-            :args=>col_aliases)
+            :args=>col_aliases, union_all: opts[:union_all])
           end
         dal = Array(d[:after_load])
         dal << proc do |m, descs|
@@ -299,7 +302,7 @@ module Sequel
           table_alias = model.dataset.schema_and_table(model.table_name)[1].to_sym
           ds = model.from(SQL::AliasedExpression.new(t, table_alias)).
             with_recursive(t, base_case, recursive_case,
-              :args=>((key_aliases + col_aliases + (level ? [la] : [])) if col_aliases))
+              :args=>((key_aliases + col_aliases + (level ? [la] : [])) if col_aliases), union_all: opts[:union_all])
           ds = r.apply_eager_dataset_changes(ds)
           ds = ds.select_append(ka) unless ds.opts[:select] == nil
           model.eager_load_results(r, eo.merge(:loader=>false, :initialize_rows=>false, :dataset=>ds, :id_map=>nil, :associations=>OPTS)) do |obj|

data/lib/sequel/version.rb

@@ -6,7 +6,7 @@ module Sequel
 
   # The minor version of Sequel.  Bumped for every non-patch level
   # release, generally around once a month.
-  MINOR = 74
+  MINOR = 76
 
   # The tiny version of Sequel.  Usually 0, only bumped for bugfix
   # releases that fix regressions from previous versions.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequel
 version: !ruby/object:Gem::Version
-  version: 5.74.0
+  version: 5.76.0
 platform: ruby
 authors:
 - Jeremy Evans
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-01 00:00:00.000000000 Z
+date: 2024-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -221,6 +221,8 @@ extra_rdoc_files:
 - doc/release_notes/5.72.0.txt
 - doc/release_notes/5.73.0.txt
 - doc/release_notes/5.74.0.txt
+- doc/release_notes/5.75.0.txt
+- doc/release_notes/5.76.0.txt
 - doc/release_notes/5.8.0.txt
 - doc/release_notes/5.9.0.txt
 files:
@@ -323,6 +325,8 @@ files:
 - doc/release_notes/5.72.0.txt
 - doc/release_notes/5.73.0.txt
 - doc/release_notes/5.74.0.txt
+- doc/release_notes/5.75.0.txt
+- doc/release_notes/5.76.0.txt
 - doc/release_notes/5.8.0.txt
 - doc/release_notes/5.9.0.txt
 - doc/schema_modification.rdoc
@@ -424,6 +428,7 @@ files:
 - lib/sequel/extensions/any_not_empty.rb
 - lib/sequel/extensions/arbitrary_servers.rb
 - lib/sequel/extensions/async_thread_pool.rb
+- lib/sequel/extensions/auto_cast_date_and_time.rb
 - lib/sequel/extensions/auto_literal_strings.rb
 - lib/sequel/extensions/blank.rb
 - lib/sequel/extensions/caller_logging.rb
@@ -654,7 +659,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.3
 signing_key:
 specification_version: 4
 summary: The Database Toolkit for Ruby