sequel 5.33.0 → 5.58.0

data/lib/sequel/extensions/pg_range_ops.rb

@@ -1,7 +1,7 @@
 # frozen-string-literal: true
 #
 # The pg_range_ops extension adds support to Sequel's DSL to make
-# it easier to call PostgreSQL range functions and operators.
+# it easier to call PostgreSQL range and multirange functions and operators.
 #
 # To load the extension:
 #
@@ -11,10 +11,11 @@
 #
 #   r = Sequel.pg_range_op(:range)
 #
-# If you have also loaded the pg_range extension, you can use
-# Sequel.pg_range as well:
+# If you have also loaded the pg_range or pg_multirange extensions, you can use
+# Sequel.pg_range or Sequel.pg_multirange as well:
 #
 #   r = Sequel.pg_range(:range)
+#   r = Sequel.pg_multirange(:range)
 #
 # Also, on most Sequel expression objects, you can call the pg_range
 # method:
@@ -46,13 +47,25 @@
 #   r.upper_inc        # upper_inc(range)
 #   r.lower_inf        # lower_inf(range)
 #   r.upper_inf        # upper_inf(range)
+#
+# All of the above methods work for both ranges and multiranges, as long
+# as PostgreSQL supports the operation. The following methods are also
+# supported:
+#
+#   r.range_merge      # range_merge(range)
+#   r.unnest           # unnest(range)
+#   r.multirange       # multirange(range)
+#
+# +range_merge+ and +unnest+ expect the receiver to represent a multirange
+# value, while +multi_range+ expects the receiver to represent a range value.
 #   
-# See the PostgreSQL range function and operator documentation for more
+# See the PostgreSQL range and multirange function and operator documentation for more
 # details on what these functions and operators do.
 #
-# If you are also using the pg_range extension, you should load it before
-# loading this extension.  Doing so will allow you to use PGArray#op to get
-# an RangeOp, allowing you to perform range operations on range literals.
+# If you are also using the pg_range or pg_multirange extension, you should
+# load them before loading this extension.  Doing so will allow you to use
+# PGRange#op and PGMultiRange#op to get a RangeOp, allowing you to perform
+# range operations on range literals.
 #
 # Related module: Sequel::Postgres::RangeOp
 
@@ -77,9 +90,12 @@ module Sequel
         :overlaps => ["(".freeze, " && ".freeze, ")".freeze].freeze,
       }.freeze
 
-      %w'lower upper isempty lower_inc upper_inc lower_inf upper_inf'.each do |f|
+      %w'lower upper isempty lower_inc upper_inc lower_inf upper_inf unnest'.each do |f|
         class_eval("def #{f}; function(:#{f}) end", __FILE__, __LINE__)
       end
+      %w'range_merge multirange'.each do |f|
+        class_eval("def #{f}; RangeOp.new(function(:#{f})) end", __FILE__, __LINE__)
+      end
       OPERATORS.each_key do |f|
         class_eval("def #{f}(v); operator(:#{f}, v) end", __FILE__, __LINE__)
       end
@@ -116,7 +132,9 @@ module Sequel
       end
     end
 
+    # :nocov:
     if defined?(PGRange)
+    # :nocov:
       class PGRange
         # Wrap the PGRange instance in an RangeOp, allowing you to easily use
         # the PostgreSQL range functions and operators with literal ranges.
@@ -125,6 +143,18 @@ module Sequel
         end
       end
     end
+
+    # :nocov:
+    if defined?(PGMultiRange)
+    # :nocov:
+      class PGMultiRange
+        # Wrap the PGRange instance in an RangeOp, allowing you to easily use
+        # the PostgreSQL range functions and operators with literal ranges.
+        def op
+          RangeOp.new(self)
+        end
+      end
+    end
   end
 
   module SQL::Builders
@@ -158,7 +188,7 @@ end
 if defined?(Sequel::CoreRefinements)
   module Sequel::CoreRefinements
     refine Symbol do
-      include Sequel::Postgres::RangeOpMethods
+      send INCLUDE_METH, Sequel::Postgres::RangeOpMethods
     end
   end
 end

data/lib/sequel/extensions/pg_row.rb

@@ -222,7 +222,6 @@ module Sequel
         # Split the stored string into an array of strings, handling
         # the different types of quoting.
         def parse
-          return @result if @result
           values = []
           skip(/\(/)
           if skip(/\)/)
@@ -483,6 +482,7 @@ module Sequel
               row_type(db_type, v)
             end
             private meth
+            alias_method(meth, meth)
           end
 
           nil

data/lib/sequel/extensions/pg_row_ops.rb

@@ -158,6 +158,30 @@ module Sequel
         end
       end
     end
+
+    # :nocov:
+    if defined?(PGRow::ArrayRow)
+    # :nocov:
+      class PGRow::ArrayRow
+        # Wrap the PGRow::ArrayRow instance in an PGRowOp, allowing you to easily use
+        # the PostgreSQL row functions and operators with literal rows.
+        def op
+          Sequel.pg_row_op(self)
+        end
+      end
+    end
+
+    # :nocov:
+    if defined?(PGRow::HashRow)
+    # :nocov:
+      class PGRow::HashRow
+        # Wrap the PGRow::ArrayRow instance in an PGRowOp, allowing you to easily use
+        # the PostgreSQL row functions and operators with literal rows.
+        def op
+          Sequel.pg_row_op(self)
+        end
+      end
+    end
   end
 
   module SQL::Builders
@@ -186,7 +210,7 @@ end
 if defined?(Sequel::CoreRefinements)
   module Sequel::CoreRefinements
     refine Symbol do
-      include Sequel::Postgres::PGRowOp::ExpressionMethods
+      send INCLUDE_METH, Sequel::Postgres::PGRowOp::ExpressionMethods
     end
   end
 end

data/lib/sequel/extensions/query.rb

@@ -74,6 +74,9 @@ module Sequel
         raise(Sequel::Error, "method #{method.inspect} did not return a dataset") unless @dataset.is_a?(Dataset)
         self
       end
+      # :nocov:
+      ruby2_keywords(:method_missing) if respond_to?(:ruby2_keywords, true)
+      # :nocov:
     end
   end
 

data/lib/sequel/extensions/run_transaction_hooks.rb

@@ -48,7 +48,7 @@ class Sequel::Database
     def _run_transaction_hooks(type, opts)
       synchronize(opts[:server]) do |conn|
         unless h = _trans(conn)
-          raise Error, "Cannot call run_#{type}_hooks outside of a transaction"
+          raise Sequel::Error, "Cannot call run_#{type}_hooks outside of a transaction"
         end
 
         if hooks = h[type]

data/lib/sequel/extensions/s.rb

@@ -49,9 +49,12 @@ module Sequel::S
     Sequel.expr(*a, &block)
   end
 
+  # :nocov:
   if RUBY_VERSION >= '2.0.0'
+    include_meth = RUBY_VERSION >= '3.1' ? :import_methods : :include
+  # :nocov:
     refine Object do
-      include Sequel::S
+      send include_meth, Sequel::S
     end
   end
 end

data/lib/sequel/extensions/schema_dumper.rb

@@ -6,6 +6,17 @@
 # the current database).  The main interface is through
 # Sequel::Database#dump_schema_migration.
 #
+# The schema_dumper extension is quite limited in what types of
+# database objects it supports.  In general, it only supports
+# dumping tables, columns, primary key and foreign key constraints,
+# and some indexes.  It does not support most table options, CHECK
+# constraints, partial indexes, database functions, triggers,
+# security grants/revokes, and a wide variety of other useful
+# database properties.  Be aware of the limitations when using the
+# schema_dumper extension. If you are dumping the schema to restore
+# to the same database type, it is recommended to use your database's
+# dump and restore programs instead of the schema_dumper extension.
+#
 # To load the extension:
 #
 #   DB.extension :schema_dumper
@@ -37,7 +48,7 @@ module Sequel
         {:type =>schema[:type] == :boolean ? TrueClass : Integer}
       when /\Abigint(?:\((?:\d+)\))?(?: unsigned)?\z/
         {:type=>:Bignum}
-      when /\A(?:real|float(?: unsigned)?|double(?: precision)?|double\(\d+,\d+\)(?: unsigned)?)\z/
+      when /\A(?:real|float|double(?: precision)?|double\(\d+,\d+\))(?: unsigned)?\z/
         {:type=>Float}
       when 'boolean', 'bit', 'bool'
         {:type=>TrueClass}
@@ -57,7 +68,7 @@ module Sequel
         {:type=>String, :size=>($1.to_i if $1)}
       when /\A(?:small)?money\z/
         {:type=>BigDecimal, :size=>[19,2]}
-      when /\A(?:decimal|numeric|number)(?:\((\d+)(?:,\s*(\d+))?\))?\z/
+      when /\A(?:decimal|numeric|number)(?:\((\d+)(?:,\s*(\d+))?\))?(?: unsigned)?\z/
         s = [($1.to_i if $1), ($2.to_i if $2)].compact
         {:type=>BigDecimal, :size=>(s.empty? ? nil : s)}
       when /\A(?:bytea|(?:tiny|medium|long)?blob|(?:var)?binary)(?:\((\d+)\))?\z/
@@ -172,7 +183,7 @@ END_MIG
       if options[:single_pk] && schema_autoincrementing_primary_key?(schema)
         type_hash = options[:same_db] ? {:type=>schema[:db_type]} : column_schema_to_ruby_type(schema)
         [:table, :key, :on_delete, :on_update, :deferrable].each{|f| type_hash[f] = schema[f] if schema[f]}
-        if type_hash == {:type=>Integer} || type_hash == {:type=>"integer"}
+        if type_hash == {:type=>Integer} || type_hash == {:type=>"integer"} || type_hash == {:type=>"INTEGER"}
           type_hash.delete(:type)
         elsif options[:same_db] && type_hash == {:type=>type_literal_generic_bignum_symbol(type_hash).to_s}
           type_hash[:type] = :Bignum
@@ -214,11 +225,11 @@ END_MIG
         col_opts[:null] = false if schema[:allow_null] == false
         if table = schema[:table]
           [:key, :on_delete, :on_update, :deferrable].each{|f| col_opts[f] = schema[f] if schema[f]}
-          col_opts[:type] = type unless type == Integer || type == 'integer'
+          col_opts[:type] = type unless type == Integer || type == 'integer' || type == 'INTEGER'
           gen.foreign_key(name, table, col_opts)
         else
           gen.column(name, type, col_opts)
-          if [Integer, :Bignum, Float].include?(type) && schema[:db_type] =~ / unsigned\z/io
+          if [Integer, :Bignum, Float, BigDecimal].include?(type) && schema[:db_type] =~ / unsigned\z/io
             gen.check(Sequel::SQL::Identifier.new(name) >= 0)
           end
         end

data/lib/sequel/extensions/server_block.rb

@@ -88,12 +88,10 @@ module Sequel
   module UnthreadedServerBlock
     # Set a default server/shard to use inside the block.
     def with_server(default_server, read_only_server=default_server)
-      begin
-        set_default_server(default_server, read_only_server)
-        yield
-      ensure
-        clear_default_server
-      end
+      set_default_server(default_server, read_only_server)
+      yield
+    ensure
+      clear_default_server
     end
 
     private
@@ -131,12 +129,10 @@ module Sequel
     # Set a default server/shard to use inside the block for the current
     # thread.
     def with_server(default_server, read_only_server=default_server)
-      begin
-        set_default_server(default_server, read_only_server)
-        yield
-      ensure
-        clear_default_server
-      end
+      set_default_server(default_server, read_only_server)
+      yield
+    ensure
+      clear_default_server
     end
 
     private

data/lib/sequel/extensions/sql_comments.rb

@@ -44,11 +44,51 @@
 #
 #   DB.extension(:sql_comments)
 #
+# Loading the sql_comments extension into the database also adds 
+# support for block-level comment support via Database#with_comments.
+# You call #with_comments with a hash.  Queries inside the hash will
+# include a comment based on the hash (assuming they are inside the
+# same thread):
+#
+#   DB.with_comments(model: Album, action: :all) do
+#     DB[:albums].all
+#     # SELECT * FROM albums -- model:Album,action:all
+#   end
+#
+# You can nest calls to #with_comments, which will combine the
+# entries from both calls:
+#
+#   DB.with_comments(application: App, path: :scrubbed_path) do
+#     DB.with_comments(model: Album, action: :all) do
+#       ds = DB[:albums].all
+#       # SELECT * FROM albums
+#       # -- application:App,path:scrubbed_path,model:Album,action:all
+#     end
+#   end
+#
+# You can override comment entries specified in earlier blocks, or
+# remove entries specified earlier using a nil value:
+#
+#   DB.with_comments(application: App, path: :scrubbed_path) do
+#     DB.with_comments(application: Foo, path: nil) do
+#       ds = DB[:albums].all
+#       # SELECT * FROM albums # -- application:Foo
+#     end
+#   end
+#
+# You can combine block-level comments with dataset-specific
+# comments:
+#
+#   DB.with_comments(model: Album, action: :all) do
+#     DB[:table].comment("Some Comment").all
+#     # SELECT * FROM albums -- model:Album,action:all -- Some Comment
+#   end
+#
 # Note that Microsoft Access does not support inline comments,
 # and attempting to use comments on it will result in SQL syntax
 # errors.
 #
-# Related module: Sequel::SQLComments
+# Related modules: Sequel::SQLComments, Sequel::Database::SQLComments
 
 #
 module Sequel
@@ -62,7 +102,7 @@ module Sequel
     %w'select insert update delete'.each do |type|
       define_method(:"#{type}_sql") do |*a|
         sql = super(*a)
-        if comment = @opts[:comment]
+        if comment = _sql_comment
           # This assumes that the comment stored in the dataset has
           # already been formatted. If not, this could result in SQL
           # injection.
@@ -74,8 +114,10 @@ module Sequel
           if sql.frozen?
             sql += comment
             sql.freeze
-          else
+          elsif @opts[:append_sql] || @opts[:placeholder_literalizer]
             sql << comment
+          else
+            sql += comment
           end
         end
         sql
@@ -84,6 +126,11 @@ module Sequel
 
     private
 
+    # The comment to include in the SQL query, if any.
+    def _sql_comment
+      @opts[:comment]
+    end
+
     # Format the comment.  For maximum compatibility, this uses a
     # single line SQL comment, and converts all consecutive whitespace
     # in the comment to a single space.
@@ -92,5 +139,65 @@ module Sequel
     end
   end
 
+  module Database::SQLComments
+    def self.extended(db)
+      db.instance_variable_set(:@comment_hashes, {})
+      db.extend_datasets DatasetSQLComments
+    end
+
+    # A map of threads to comment hashes, used for correctly setting
+    # comments for all queries inside #with_comments blocks.
+    attr_reader :comment_hashes
+
+    # Store the comment hash and use it to create comments inside the block
+    def with_comments(comment_hash)
+      hashes = @comment_hashes
+      t = Sequel.current
+      new_hash = if hash = Sequel.synchronize{hashes[t]}
+        hash.merge(comment_hash)
+      else
+        comment_hash.dup
+      end
+      yield Sequel.synchronize{hashes[t] = new_hash}
+    ensure
+      if hash
+        Sequel.synchronize{hashes[t] = hash}
+      else
+        t && Sequel.synchronize{hashes.delete(t)}
+      end
+    end
+
+    module DatasetSQLComments
+      include Sequel::SQLComments
+
+      private
+
+      # Include comments added via Database#with_comments in the output SQL.
+      def _sql_comment
+        specific_comment = super
+        return specific_comment if @opts[:append_sql]
+
+        t = Sequel.current
+        hashes = db.comment_hashes
+        block_comment = if comment_hash = Sequel.synchronize{hashes[t]}
+          comment_array = comment_hash.map{|k,v| "#{k}:#{v}" unless v.nil?}
+          comment_array.compact!
+          comment_array.join(",")
+        end
+
+        if block_comment
+          if specific_comment
+            format_sql_comment(block_comment + specific_comment)
+          else
+            format_sql_comment(block_comment)
+          end
+        else
+          specific_comment
+        end
+      end
+    end
+  end
+
   Dataset.register_extension(:sql_comments, SQLComments)
+  Database.register_extension(:sql_comments, Database::SQLComments)
 end

data/lib/sequel/extensions/sql_log_normalizer.rb

@@ -0,0 +1,108 @@
+# frozen-string-literal: true
+#
+# The sql_log_normalizer extension normalizes the SQL that is logged,
+# removing the literal strings and numbers in the SQL, and removing the
+# logging of any bound variables:
+#
+#   ds = DB[:table].first(a: 1, b: 'something')
+#   # Without sql_log_normalizer extension
+#   # SELECT * FROM "table" WHERE (("a" = 1) AND ("b" = 'something')) LIMIT 1
+#
+#   # With sql_log_normalizer_extension
+#   # SELECT * FROM "table" WHERE (("a" = ?) AND ("b" = ?)) LIMIT ?
+#
+# The normalization is done by scanning the SQL string being executed
+# for literal strings and numbers, and replacing them with question
+# marks.  While this should work for all or almost all production queries,
+# there are pathlogical queries that will not be handled correctly, such as
+# the use of apostrophes in identifiers:
+#
+#   DB[:"asf'bar"].where(a: 1, b: 'something').first
+#   # Logged as:
+#   # SELECT * FROM "asf?something')) LIMIT ?
+#
+# The expected use case for this extension is when you want to normalize
+# logs to group similar queries, or when you want to protect sensitive
+# data from being stored in the logs.
+#
+# Related module: Sequel::SQLLogNormalizer
+
+#
+module Sequel
+  module SQLLogNormalizer
+    def self.extended(db)
+      type = case db.literal("'")
+      when "''''"
+        :standard
+      when "'\\''"
+        :backslash
+      when "N''''"
+        :n_standard
+      else
+        raise Error, "SQL log normalization is not supported on this database (' literalized as #{db.literal("'").inspect})"
+      end
+      db.instance_variable_set(:@sql_string_escape_type, type)
+    end
+
+    # Normalize the SQL before calling super.
+    def log_connection_yield(sql, conn, args=nil)
+      unless skip_logging?
+        sql = normalize_logged_sql(sql)
+        args = nil
+      end
+      super
+    end
+
+    # Replace literal strings and numbers in SQL with question mark placeholders.
+    def normalize_logged_sql(sql)
+      sql = sql.dup
+      sql.force_encoding('BINARY')
+      start_index = 0
+      check_n = @sql_string_escape_type == :n_standard
+      outside_string = true
+
+      if @sql_string_escape_type == :backslash
+        search_char = /[\\']/
+        escape_char_offset = 0
+        escape_char_value = 92 # backslash
+      else
+        search_char = "'"
+        escape_char_offset = 1
+        escape_char_value = 39 # apostrophe
+      end
+
+      # The approach used here goes against Sequel's philosophy of never attempting
+      # to parse SQL.  However, parsing the SQL is basically the only way to implement
+      # this support with Sequel's design, and it's better to be pragmatic and accept
+      # this than not be able to support this.
+
+      # Replace literal strings
+      while outside_string && (index = start_index = sql.index("'", start_index))
+        if check_n && index != 0 && sql.getbyte(index-1) == 78 # N' start
+          start_index -= 1
+        end
+        index += 1
+        outside_string = false
+
+        while (index = sql.index(search_char, index)) && (sql.getbyte(index + escape_char_offset) == escape_char_value)
+          # skip escaped characters inside string literal
+          index += 2
+        end
+
+        if index
+          # Found end of string
+          sql[start_index..index] = '?'
+          start_index += 1
+          outside_string = true
+        end
+      end
+
+      # Replace integer and decimal floating point numbers
+      sql.gsub!(/\b-?\d+(?:\.\d+)?\b/, '?')
+
+      sql
+    end
+  end
+
+  Database.register_extension(:sql_log_normalizer, SQLLogNormalizer)
+end