sequel 5.39.0 → 5.63.0

data/lib/sequel/extensions/sql_comments.rb

@@ -44,11 +44,51 @@
 #
 #   DB.extension(:sql_comments)
 #
+# Loading the sql_comments extension into the database also adds 
+# support for block-level comment support via Database#with_comments.
+# You call #with_comments with a hash.  Queries inside the hash will
+# include a comment based on the hash (assuming they are inside the
+# same thread):
+#
+#   DB.with_comments(model: Album, action: :all) do
+#     DB[:albums].all
+#     # SELECT * FROM albums -- model:Album,action:all
+#   end
+#
+# You can nest calls to #with_comments, which will combine the
+# entries from both calls:
+#
+#   DB.with_comments(application: App, path: :scrubbed_path) do
+#     DB.with_comments(model: Album, action: :all) do
+#       ds = DB[:albums].all
+#       # SELECT * FROM albums
+#       # -- application:App,path:scrubbed_path,model:Album,action:all
+#     end
+#   end
+#
+# You can override comment entries specified in earlier blocks, or
+# remove entries specified earlier using a nil value:
+#
+#   DB.with_comments(application: App, path: :scrubbed_path) do
+#     DB.with_comments(application: Foo, path: nil) do
+#       ds = DB[:albums].all
+#       # SELECT * FROM albums # -- application:Foo
+#     end
+#   end
+#
+# You can combine block-level comments with dataset-specific
+# comments:
+#
+#   DB.with_comments(model: Album, action: :all) do
+#     DB[:table].comment("Some Comment").all
+#     # SELECT * FROM albums -- model:Album,action:all -- Some Comment
+#   end
+#
 # Note that Microsoft Access does not support inline comments,
 # and attempting to use comments on it will result in SQL syntax
 # errors.
 #
-# Related module: Sequel::SQLComments
+# Related modules: Sequel::SQLComments, Sequel::Database::SQLComments
 
 #
 module Sequel
@@ -62,7 +102,7 @@ module Sequel
     %w'select insert update delete'.each do |type|
       define_method(:"#{type}_sql") do |*a|
         sql = super(*a)
-        if comment = @opts[:comment]
+        if comment = _sql_comment
           # This assumes that the comment stored in the dataset has
           # already been formatted. If not, this could result in SQL
           # injection.
@@ -74,8 +114,10 @@ module Sequel
           if sql.frozen?
             sql += comment
             sql.freeze
-          else
+          elsif @opts[:append_sql] || @opts[:placeholder_literalizer]
             sql << comment
+          else
+            sql += comment
           end
         end
         sql
@@ -84,6 +126,11 @@ module Sequel
 
     private
 
+    # The comment to include in the SQL query, if any.
+    def _sql_comment
+      @opts[:comment]
+    end
+
     # Format the comment.  For maximum compatibility, this uses a
     # single line SQL comment, and converts all consecutive whitespace
     # in the comment to a single space.
@@ -92,5 +139,65 @@ module Sequel
     end
   end
 
+  module Database::SQLComments
+    def self.extended(db)
+      db.instance_variable_set(:@comment_hashes, {})
+      db.extend_datasets DatasetSQLComments
+    end
+
+    # A map of threads to comment hashes, used for correctly setting
+    # comments for all queries inside #with_comments blocks.
+    attr_reader :comment_hashes
+
+    # Store the comment hash and use it to create comments inside the block
+    def with_comments(comment_hash)
+      hashes = @comment_hashes
+      t = Sequel.current
+      new_hash = if hash = Sequel.synchronize{hashes[t]}
+        hash.merge(comment_hash)
+      else
+        comment_hash.dup
+      end
+      yield Sequel.synchronize{hashes[t] = new_hash}
+    ensure
+      if hash
+        Sequel.synchronize{hashes[t] = hash}
+      else
+        t && Sequel.synchronize{hashes.delete(t)}
+      end
+    end
+
+    module DatasetSQLComments
+      include Sequel::SQLComments
+
+      private
+
+      # Include comments added via Database#with_comments in the output SQL.
+      def _sql_comment
+        specific_comment = super
+        return specific_comment if @opts[:append_sql]
+
+        t = Sequel.current
+        hashes = db.comment_hashes
+        block_comment = if comment_hash = Sequel.synchronize{hashes[t]}
+          comment_array = comment_hash.map{|k,v| "#{k}:#{v}" unless v.nil?}
+          comment_array.compact!
+          comment_array.join(",")
+        end
+
+        if block_comment
+          if specific_comment
+            format_sql_comment(block_comment + specific_comment)
+          else
+            format_sql_comment(block_comment)
+          end
+        else
+          specific_comment
+        end
+      end
+    end
+  end
+
   Dataset.register_extension(:sql_comments, SQLComments)
+  Database.register_extension(:sql_comments, Database::SQLComments)
 end

data/lib/sequel/extensions/sql_log_normalizer.rb

@@ -0,0 +1,108 @@
+# frozen-string-literal: true
+#
+# The sql_log_normalizer extension normalizes the SQL that is logged,
+# removing the literal strings and numbers in the SQL, and removing the
+# logging of any bound variables:
+#
+#   ds = DB[:table].first(a: 1, b: 'something')
+#   # Without sql_log_normalizer extension
+#   # SELECT * FROM "table" WHERE (("a" = 1) AND ("b" = 'something')) LIMIT 1
+#
+#   # With sql_log_normalizer_extension
+#   # SELECT * FROM "table" WHERE (("a" = ?) AND ("b" = ?)) LIMIT ?
+#
+# The normalization is done by scanning the SQL string being executed
+# for literal strings and numbers, and replacing them with question
+# marks.  While this should work for all or almost all production queries,
+# there are pathlogical queries that will not be handled correctly, such as
+# the use of apostrophes in identifiers:
+#
+#   DB[:"asf'bar"].where(a: 1, b: 'something').first
+#   # Logged as:
+#   # SELECT * FROM "asf?something')) LIMIT ?
+#
+# The expected use case for this extension is when you want to normalize
+# logs to group similar queries, or when you want to protect sensitive
+# data from being stored in the logs.
+#
+# Related module: Sequel::SQLLogNormalizer
+
+#
+module Sequel
+  module SQLLogNormalizer
+    def self.extended(db)
+      type = case db.literal("'")
+      when "''''"
+        :standard
+      when "'\\''"
+        :backslash
+      when "N''''"
+        :n_standard
+      else
+        raise Error, "SQL log normalization is not supported on this database (' literalized as #{db.literal("'").inspect})"
+      end
+      db.instance_variable_set(:@sql_string_escape_type, type)
+    end
+
+    # Normalize the SQL before calling super.
+    def log_connection_yield(sql, conn, args=nil)
+      unless skip_logging?
+        sql = normalize_logged_sql(sql)
+        args = nil
+      end
+      super
+    end
+
+    # Replace literal strings and numbers in SQL with question mark placeholders.
+    def normalize_logged_sql(sql)
+      sql = sql.dup
+      sql.force_encoding('BINARY')
+      start_index = 0
+      check_n = @sql_string_escape_type == :n_standard
+      outside_string = true
+
+      if @sql_string_escape_type == :backslash
+        search_char = /[\\']/
+        escape_char_offset = 0
+        escape_char_value = 92 # backslash
+      else
+        search_char = "'"
+        escape_char_offset = 1
+        escape_char_value = 39 # apostrophe
+      end
+
+      # The approach used here goes against Sequel's philosophy of never attempting
+      # to parse SQL.  However, parsing the SQL is basically the only way to implement
+      # this support with Sequel's design, and it's better to be pragmatic and accept
+      # this than not be able to support this.
+
+      # Replace literal strings
+      while outside_string && (index = start_index = sql.index("'", start_index))
+        if check_n && index != 0 && sql.getbyte(index-1) == 78 # N' start
+          start_index -= 1
+        end
+        index += 1
+        outside_string = false
+
+        while (index = sql.index(search_char, index)) && (sql.getbyte(index + escape_char_offset) == escape_char_value)
+          # skip escaped characters inside string literal
+          index += 2
+        end
+
+        if index
+          # Found end of string
+          sql[start_index..index] = '?'
+          start_index += 1
+          outside_string = true
+        end
+      end
+
+      # Replace integer and decimal floating point numbers
+      sql.gsub!(/\b-?\d+(?:\.\d+)?\b/, '?')
+
+      sql
+    end
+  end
+
+  Database.register_extension(:sql_log_normalizer, SQLLogNormalizer)
+end

data/lib/sequel/extensions/sqlite_json_ops.rb

@@ -0,0 +1,255 @@
+# frozen-string-literal: true
+#
+# The sqlite_json_ops extension adds support to Sequel's DSL to make
+# it easier to call SQLite JSON functions and operators (added
+# first in SQLite 3.38.0).
+#
+# To load the extension:
+#
+#   Sequel.extension :sqlite_json_ops
+#
+# This extension works by calling methods on Sequel::SQLite::JSONOp objects,
+# which you can create via Sequel.sqlite_json_op:
+#
+#   j = Sequel.sqlite_json_op(:json_column)
+#
+# Also, on most Sequel expression objects, you can call the sqlite_json_op method
+# to create a Sequel::SQLite::JSONOp object:
+#
+#   j = Sequel[:json_column].sqlite_json_op
+#
+# If you have loaded the {core_extensions extension}[rdoc-ref:doc/core_extensions.rdoc],
+# or you have loaded the core_refinements extension
+# and have activated refinements for the file, you can also use Symbol#sqlite_json_op:
+#
+#   j = :json_column.sqlite_json_op
+#
+# The following methods are available for Sequel::SQLite::JSONOp instances:
+#
+#   j[1]                     # (json_column ->> 1)
+#   j.get(1)                 # (json_column ->> 1)
+#   j.get_text(1)            # (json_column -> 1)
+#   j.extract('$.a')         # json_extract(json_column, '$.a')
+#
+#   j.array_length           # json_array_length(json_column)
+#   j.type                   # json_type(json_column)
+#   j.valid                  # json_valid(json_column)
+#   j.json                   # json(json_column)
+#
+#   j.insert('$.a', 1)       # json_insert(json_column, '$.a', 1)
+#   j.set('$.a', 1)          # json_set(json_column, '$.a', 1)
+#   j.replace('$.a', 1)      # json_replace(json_column, '$.a', 1)
+#   j.remove('$.a')          # json_remove(json_column, '$.a')
+#   j.patch('{"a":2}')       # json_patch(json_column, '{"a":2}')
+#
+#   j.each                   # json_each(json_column)
+#   j.tree                   # json_tree(json_column)
+#
+# Related modules: Sequel::SQLite::JSONOp
+
+#
+module Sequel
+  module SQLite
+    # The JSONOp class is a simple container for a single object that
+    # defines methods that yield Sequel expression objects representing
+    # SQLite json operators and functions.
+    #
+    # In the method documentation examples, assume that:
+    #
+    #   json_op = Sequel.sqlite_json_op(:json)
+    class JSONOp < Sequel::SQL::Wrapper
+      GET = ["(".freeze, " ->> ".freeze, ")".freeze].freeze
+      private_constant :GET
+
+      GET_JSON = ["(".freeze, " -> ".freeze, ")".freeze].freeze
+      private_constant :GET_JSON
+
+      # Returns an expression for getting the JSON array element or object field
+      # at the specified path as a SQLite value.
+      #
+      #   json_op[1]         # (json ->> 1)
+      #   json_op['a']       # (json ->> 'a')
+      #   json_op['$.a.b']   # (json ->> '$.a.b')
+      #   json_op['$[1][2]'] # (json ->> '$[1][2]')
+      def [](key)
+        json_op(GET, key)
+      end
+      alias get []
+
+      # Returns an expression for the length of the JSON array, or the JSON array at
+      # the given path.
+      #
+      #   json_op.array_length         # json_array_length(json)
+      #   json_op.array_length('$[1]') # json_array_length(json, '$[1]')
+      def array_length(*args)
+        Sequel::SQL::NumericExpression.new(:NOOP, function(:array_length, *args))
+      end
+
+      # Returns an expression for a set of information extracted from the top-level
+      # members of the JSON array or object, or the top-level members of the JSON array
+      # or object at the given path.
+      #
+      #   json_op.each        # json_each(json)
+      #   json_op.each('$.a') # json_each(json, '$.a')
+      def each(*args)
+        function(:each, *args)
+      end
+
+      # Returns an expression for the JSON array element or object field at the specified
+      # path as a SQLite value, but only accept paths as arguments, and allow the use of
+      # multiple paths.
+      #
+      #   json_op.extract('$.a')        # json_extract(json, '$.a')
+      #   json_op.extract('$.a', '$.b') # json_extract(json, '$.a', '$.b')
+      def extract(*a)
+        function(:extract, *a)
+      end
+
+      # Returns an expression for getting the JSON array element or object field at the
+      # specified path as a JSON value.
+      #
+      #   json_op.get_json(1)         # (json -> 1)
+      #   json_op.get_json('a')       # (json -> 'a')
+      #   json_op.get_json('$.a.b')   # (json -> '$.a.b')
+      #   json_op.get_json('$[1][2]') # (json -> '$[1][2]')
+      def get_json(key)
+        self.class.new(json_op(GET_JSON, key))
+      end
+
+      # Returns an expression for creating new entries at the given paths in the JSON array
+      # or object, but not overwriting existing entries.
+      #
+      #   json_op.insert('$.a', 1)           # json_insert(json, '$.a', 1)
+      #   json_op.insert('$.a', 1, '$.b', 2) # json_insert(json, '$.a', 1, '$.b', 2)
+      def insert(path, value, *args)
+        wrapped_function(:insert, path, value, *args)
+      end
+
+      # Returns an expression for a minified version of the JSON.
+      #
+      #   json_op.json   # json(json)
+      def json
+        self.class.new(SQL::Function.new(:json, self))
+      end
+      alias minify json
+
+      # Returns an expression for updating the JSON object using the RFC 7396 MergePatch algorithm
+      #
+      #   json_op.patch('{"a": 1, "b": null}') # json_patch(json, '{"a": 1, "b": null}')
+      def patch(json_patch)
+        wrapped_function(:patch, json_patch)
+      end
+
+      # Returns an expression for removing entries at the given paths from the JSON array or object.
+      #
+      #   json_op.remove('$.a')        # json_remove(json, '$.a')
+      #   json_op.remove('$.a', '$.b') # json_remove(json, '$.a', '$.b')
+      def remove(path, *paths)
+        wrapped_function(:remove, path, *paths)
+      end
+
+      # Returns an expression for replacing entries at the given paths in the JSON array or object,
+      # but not creating new entries.
+      #
+      #   json_op.replace('$.a', 1)           # json_replace(json, '$.a', 1)
+      #   json_op.replace('$.a', 1, '$.b', 2) # json_replace(json, '$.a', 1, '$.b', 2)
+      def replace(path, value, *args)
+        wrapped_function(:replace, path, value, *args)
+      end
+
+      # Returns an expression for creating or replacing entries at the given paths in the
+      # JSON array or object.
+      #
+      #   json_op.set('$.a', 1)           # json_set(json, '$.a', 1)
+      #   json_op.set('$.a', 1, '$.b', 2) # json_set(json, '$.a', 1, '$.b', 2)
+      def set(path, value, *args)
+        wrapped_function(:set, path, value, *args)
+      end
+
+      # Returns an expression for a set of information extracted from the JSON array or object, or
+      # the JSON array or object at the given path.
+      #
+      #   json_op.tree        # json_tree(json)
+      #   json_op.tree('$.a') # json_tree(json, '$.a')
+      def tree(*args)
+        function(:tree, *args)
+      end
+
+      # Returns an expression for the type of the JSON value or the JSON value at the given path.
+      #
+      #   json_op.type         # json_type(json)
+      #   json_op.type('$[1]') # json_type(json, '$[1]')
+      def type(*args)
+        Sequel::SQL::StringExpression.new(:NOOP, function(:type, *args))
+      end
+      alias typeof type
+
+      # Returns a boolean expression for whether the JSON is valid or not.
+      def valid
+        Sequel::SQL::BooleanExpression.new(:NOOP, function(:valid))
+      end
+
+      private
+
+      # Internals of the [], get, get_json methods, using a placeholder literal string.
+      def json_op(str, args)
+        self.class.new(Sequel::SQL::PlaceholderLiteralString.new(str, [self, args]))
+      end
+
+      # Internals of the methods that return functions prefixed with +json_+.
+      def function(name, *args)
+        SQL::Function.new("json_#{name}", self, *args)
+      end
+
+      # Internals of the methods that return functions prefixed with +json_+, that
+      # return JSON values.
+      def wrapped_function(*args)
+        self.class.new(function(*args))
+      end
+    end
+
+    module JSONOpMethods
+      # Wrap the receiver in an JSONOp so you can easily use the SQLite
+      # json functions and operators with it.
+      def sqlite_json_op
+        JSONOp.new(self)
+      end
+    end
+  end
+
+  module SQL::Builders
+    # Return the object wrapped in an SQLite::JSONOp.
+    def sqlite_json_op(v)
+      case v
+      when SQLite::JSONOp
+        v
+      else
+        SQLite::JSONOp.new(v)
+      end
+    end
+  end
+
+  class SQL::GenericExpression
+    include Sequel::SQLite::JSONOpMethods
+  end
+
+  class LiteralString
+    include Sequel::SQLite::JSONOpMethods
+  end
+end
+
+# :nocov:
+if Sequel.core_extensions?
+  class Symbol
+    include Sequel::SQLite::JSONOpMethods
+  end
+end
+
+if defined?(Sequel::CoreRefinements)
+  module Sequel::CoreRefinements
+    refine Symbol do
+      send INCLUDE_METH, Sequel::SQLite::JSONOpMethods
+    end
+  end
+end
+# :nocov:

data/lib/sequel/extensions/string_agg.rb

@@ -147,7 +147,7 @@ module Sequel
       def initialize(expr, separator=nil)
         @expr = expr
         @separator = separator
-        yield self if block_given?
+        yield self if defined?(yield)
         freeze
       end
 

data/lib/sequel/extensions/string_date_time.rb

@@ -4,6 +4,10 @@
 # for converting the strings to a date (e.g. String#to_date), allowing
 # for backwards compatibility with legacy Sequel code.
 #
+# These methods calls +parse+ on the related class, and as such, can
+# result in denial of service in older versions of Ruby for large
+# untrusted input, and raise exceptions in newer versions of Ruby. 
+#
 # To load the extension:
 #
 #   Sequel.extension :string_date_time
@@ -11,42 +15,34 @@
 class String
   # Converts a string into a Date object.
   def to_date
-    begin
-      Date.parse(self, Sequel.convert_two_digit_years)
-    rescue => e
-      raise Sequel.convert_exception_class(e, Sequel::InvalidValue)
-    end
+    Date.parse(self, Sequel.convert_two_digit_years)
+  rescue => e
+    raise Sequel.convert_exception_class(e, Sequel::InvalidValue)
   end
 
   # Converts a string into a DateTime object.
   def to_datetime
-    begin
-      DateTime.parse(self, Sequel.convert_two_digit_years)
-    rescue => e
-      raise Sequel.convert_exception_class(e, Sequel::InvalidValue)
-    end
+    DateTime.parse(self, Sequel.convert_two_digit_years)
+  rescue => e
+    raise Sequel.convert_exception_class(e, Sequel::InvalidValue)
   end
 
   # Converts a string into a Time or DateTime object, depending on the
   # value of Sequel.datetime_class
   def to_sequel_time
-    begin
-      if Sequel.datetime_class == DateTime
-        DateTime.parse(self, Sequel.convert_two_digit_years)
-      else
-        Sequel.datetime_class.parse(self)
-      end
-    rescue => e
-      raise Sequel.convert_exception_class(e, Sequel::InvalidValue)
+    if Sequel.datetime_class == DateTime
+      DateTime.parse(self, Sequel.convert_two_digit_years)
+    else
+      Sequel.datetime_class.parse(self)
     end
+  rescue => e
+    raise Sequel.convert_exception_class(e, Sequel::InvalidValue)
   end
 
   # Converts a string into a Time object.
   def to_time
-    begin
-      Time.parse(self)
-    rescue => e
-      raise Sequel.convert_exception_class(e, Sequel::InvalidValue)
-    end
+    Time.parse(self)
+  rescue => e
+    raise Sequel.convert_exception_class(e, Sequel::InvalidValue)
   end
 end

data/lib/sequel/extensions/symbol_aref.rb

@@ -35,6 +35,7 @@ if RUBY_VERSION >= '2.0'
   class Symbol
     prepend Sequel::SymbolAref
   end
+# :nocov:
 else
   class Symbol
     if method_defined?(:[])
@@ -51,3 +52,4 @@ else
     end
   end
 end
+# :nocov: