sequel 5.80.0 → 5.92.0

data/lib/sequel/extensions/pg_row.rb

@@ -113,6 +113,7 @@ module Sequel
         # automatically casted to the database type when literalizing.
         def self.subclass(db_type)
           Class.new(self) do
+            Sequel.set_temp_name(self){"Sequel::Postgres::PGRow::ArrayRow::_Subclass(#{db_type})"}
             @db_type = db_type
           end
         end
@@ -170,6 +171,7 @@ module Sequel
         # type and columns.
         def self.subclass(db_type, columns)
           Class.new(self) do
+            Sequel.set_temp_name(self){"Sequel::Postgres::PGRow::HashRow::_Subclass(#{db_type})"}
             @db_type = db_type
             @columns = columns
           end
@@ -391,7 +393,7 @@ module Sequel
           db.instance_exec do
             @row_types = {}
             @row_schema_types = {}
-            extend(@row_type_method_module = Module.new)
+            extend(@row_type_method_module = Sequel.set_temp_name(Module.new){"Sequel::Postgres::PGRow::DatabaseMethods::_RowTypeMethodModule"})
             add_conversion_proc(2249, PGRow::Parser.new(:converter=>PGRow::ArrayRow))
             if respond_to?(:register_array_type)
               register_array_type('record', :oid=>2287, :scalar_oid=>2249)

data/lib/sequel/extensions/pg_schema_caching.rb

@@ -0,0 +1,90 @@
+# frozen-string-literal: true
+#
+# The pg_schema_caching extension builds on top of the schema_caching
+# extension, and allows it to handle custom PostgreSQL types. On
+# PostgreSQL, column schema hashes include an :oid entry for the OID
+# for the column's type.  For custom types, this OID is dependent on
+# the PostgreSQL database, so in most cases, test and development
+# versions of the same database, created with the same migrations,
+# will have different OIDs.
+#
+# To fix this case, the pg_schema_caching extension removes custom
+# OIDs from the schema cache when dumping the schema, replacing them
+# with a placeholder. When loading the cached schema, the Database
+# object makes a single query to get the OIDs for all custom types
+# used by the cached schema, and it updates all related column
+# schema hashes to set the correct :oid entry for the current
+# database.
+#
+# Related module: Sequel::Postgres::SchemaCaching
+
+require_relative "schema_caching"
+
+module Sequel
+  module Postgres
+    module SchemaCaching
+      include Sequel::SchemaCaching
+
+      private
+
+      # Load custom oids from database when loading schema cache file.
+      def load_schema_cache_file(file)
+        set_custom_oids_for_cached_schema(super)
+      end
+
+      # Find all column schema hashes that use custom types.
+      # Load the oids for custom types in a single query, and update
+      # each related column schema hash with the correct oid.
+      def set_custom_oids_for_cached_schema(schemas)
+        custom_oid_rows = {}
+
+        schemas.each_value do |cols|
+          cols.each do |_, h|
+            if h[:oid] == :custom
+              (custom_oid_rows[h[:db_type]] ||= []) << h
+            end
+          end
+        end
+
+        unless custom_oid_rows.empty?
+          from(:pg_type).where(:typname=>custom_oid_rows.keys).select_hash(:typname, :oid).each do |name, oid|
+            custom_oid_rows.delete(name).each do |row|
+              row[:oid] = oid
+            end
+          end
+        end
+
+        unless custom_oid_rows.empty?
+          warn "Could not load OIDs for the following custom types: #{custom_oid_rows.keys.sort.join(", ")}", uplevel: 3
+
+          schemas.keys.each do |k|
+            if schemas[k].any?{|_,h| h[:oid] == :custom}
+              # Remove schema entry for table, so it will be queried at runtime to get the correct oids
+              schemas.delete(k)
+            end
+          end
+        end
+
+        schemas
+      end
+
+      # Replace :oid entries for custom types with :custom.
+      def dumpable_schema_cache
+        sch = super
+
+        sch.each_value do |cols|
+          cols.each do |_, h|
+            if (oid = h[:oid]) && oid >= 10000
+              h[:oid] = :custom
+            end
+          end
+        end
+
+        sch
+      end
+    end
+  end
+
+  Database.register_extension(:pg_schema_caching, Postgres::SchemaCaching)
+end
+

data/lib/sequel/extensions/provenance.rb

@@ -38,6 +38,7 @@ module Sequel
     module Provenance
       SEQUEL_LIB_PATH = (File.expand_path('../../..', __FILE__) + '/').freeze
       RUBY_STDLIB = RbConfig::CONFIG["rubylibdir"]
+      INTERNAL = '<internal'
 
       if TRUE_FREEZE
         # Include provenance information when cloning datasets.
@@ -98,6 +99,7 @@ module Sequel
         caller.find do |line|
           !(line.start_with?(SEQUEL_LIB_PATH) ||
             line.start_with?(RUBY_STDLIB) ||
+            line.start_with?(INTERNAL) ||
             (ignore && line =~ ignore))
         end
       end

data/lib/sequel/extensions/query_blocker.rb

@@ -0,0 +1,172 @@
+# frozen-string-literal: true
+#
+# The query_blocker extension adds Database#block_queries.
+# Inside the block passed to #block_queries, any attempts to
+# execute a query/statement on the database will raise a
+# Sequel::QueryBlocker::BlockedQuery exception.
+#
+#   DB.extension :query_blocker
+#   DB.block_queries do
+#     ds = DB[:table]          # No exception
+#     ds = ds.where(column: 1) # No exception
+#     ds.all                   # Exception raised
+#   end
+#
+# To handle concurrency, you can pass a :scope option:
+#
+#   # Current Thread
+#   DB.block_queries(scope: :thread){}
+#
+#   # Current Fiber
+#   DB.block_queries(scope: :fiber){}
+#
+#   # Specific Thread
+#   DB.block_queries(scope: Thread.current){}
+#
+#   # Specific Fiber
+#   DB.block_queries(scope: Fiber.current){}
+#
+# Database#block_queries is useful for blocking queries inside
+# the block.  However, there may be cases where you want to
+# allow queries in specific places inside a block_queries block.
+# You can use Database#allow_queries for that:
+#
+#   DB.block_queries do
+#     DB.allow_queries do
+#       DB[:table].all           # Query allowed
+#     end
+#
+#     DB[:table].all           # Exception raised
+#   end
+#
+# When mixing block_queries and allow_queries with scopes, the
+# narrowest scope has priority.  So if you are blocking with
+# :thread scope, and allowing with :fiber scope, queries in the
+# current fiber will be allowed, but queries in different fibers of
+# the current thread will be blocked.
+#
+# Note that this should catch all queries executed through the
+# Database instance.  Whether it catches queries executed directly
+# on a connection object depends on the adapter in use.
+#
+# Related module: Sequel::QueryBlocker
+
+require "fiber"
+
+#
+module Sequel
+  module QueryBlocker
+    # Exception class raised if there is an attempt to execute a 
+    # query/statement on the database inside a block passed to
+    # block_queries.
+    class BlockedQuery < Sequel::Error
+    end
+
+    def self.extended(db)
+      db.instance_exec do
+        @blocked_query_scopes ||= {}
+      end
+    end
+
+    # If checking a connection for validity, and a BlockedQuery exception is
+    # raised, treat it as a valid connection.  You cannot check whether the
+    # connection is valid without issuing a query, and if queries are blocked,
+    # you need to assume it is valid or assume it is not.  Since it most cases
+    # it will be valid, this assumes validity.
+    def valid_connection?(conn)
+      super
+    rescue BlockedQuery
+      true
+    end
+
+    # Check whether queries are blocked before executing them.
+    def log_connection_yield(sql, conn, args=nil)
+      # All database adapters should be calling this method around
+      # query execution (otherwise the queries would not get logged),
+      # ensuring the blocking is checked.  Any database adapter issuing
+      # a query without calling this method is considered buggy.
+      check_blocked_queries!
+      super
+    end
+
+    # Whether queries are currently blocked.
+    def block_queries?
+      b = @blocked_query_scopes
+      b.fetch(Fiber.current) do
+        b.fetch(Thread.current) do
+          b.fetch(:global, false)
+        end
+      end
+    end
+
+    # Allow queries inside the block.  Only useful if they are already blocked
+    # for the same scope. Useful for blocking queries generally, and only allowing
+    # them in specific places.  Takes the same :scope option as #block_queries.
+    def allow_queries(opts=OPTS, &block)
+      _allow_or_block_queries(false, opts, &block)
+    end
+
+    # Reject (raise an BlockedQuery exception) if there is an attempt to execute
+    # a query/statement inside the block.
+    #
+    # The :scope option indicates which queries are rejected inside the block:
+    #
+    # :global :: This is the default, and rejects all queries.
+    # :thread :: Reject all queries in the current thread.
+    # :fiber :: Reject all queries in the current fiber.
+    # Thread :: Reject all queries in the given thread.
+    # Fiber :: Reject all queries in the given fiber.
+    def block_queries(opts=OPTS, &block)
+      _allow_or_block_queries(true, opts, &block)
+    end
+    
+    private
+
+    # Internals of block_queries and allow_queries.
+    def _allow_or_block_queries(value, opts)
+      scope = query_blocker_scope(opts)
+      prev_value = nil
+      scopes = @blocked_query_scopes
+
+      begin
+        Sequel.synchronize do
+          prev_value = scopes[scope]
+          scopes[scope] = value
+        end
+
+        yield
+      ensure
+        Sequel.synchronize do
+          if prev_value.nil?
+            scopes.delete(scope)
+          else
+            scopes[scope] = prev_value
+          end
+        end
+      end
+    end
+
+    # The scope for the query block, either :global, or a Thread or Fiber instance.
+    def query_blocker_scope(opts)
+      case scope = opts[:scope]
+      when nil
+        :global
+      when :global, Thread, Fiber
+        scope
+      when :thread
+        Thread.current
+      when :fiber
+        Fiber.current
+      else
+        raise Sequel::Error, "invalid scope given to block_queries: #{scope.inspect}"
+      end
+    end
+
+    # Raise a BlockQuery exception if queries are currently blocked.
+    def check_blocked_queries!
+      raise BlockedQuery, "cannot execute query inside a block_queries block" if block_queries?
+    end
+  end
+
+  Database.register_extension(:query_blocker, QueryBlocker)
+end

data/lib/sequel/extensions/schema_caching.rb

@@ -51,14 +51,7 @@ module Sequel
   module SchemaCaching
     # Dump the cached schema to the filename given in Marshal format.
     def dump_schema_cache(file)
-      sch = {}
-      @schemas.sort.each do |k,v|
-        sch[k] = v.map do |c, h|
-          h = Hash[h]
-          h.delete(:callable_default)
-          [c, h]
-        end
-      end
+      sch = dumpable_schema_cache
       File.open(file, 'wb'){|f| f.write(Marshal.dump(sch))}
       nil
     end
@@ -72,7 +65,7 @@ module Sequel
     # Replace the schema cache with the data from the given file, which
     # should be in Marshal format.
     def load_schema_cache(file)
-      @schemas = Marshal.load(File.read(file))
+      @schemas = load_schema_cache_file(file)
       @schemas.each_value{|v| schema_post_process(v)}
       nil
     end
@@ -82,6 +75,28 @@ module Sequel
     def load_schema_cache?(file)
       load_schema_cache(file) if File.exist?(file)
     end
+
+    private
+
+    # Return the deserialized schema cache file.
+    def load_schema_cache_file(file)
+      Marshal.load(File.read(file))
+    end
+
+    # A dumpable version of the schema cache.
+    def dumpable_schema_cache
+      sch = {}
+
+      @schemas.sort.each do |k,v|
+        sch[k] = v.map do |c, h|
+          h = Hash[h]
+          h.delete(:callable_default)
+          [c, h]
+        end
+      end
+
+      sch
+    end
   end
 
   Database.register_extension(:schema_caching, SchemaCaching)

data/lib/sequel/extensions/schema_dumper.rb

@@ -27,6 +27,11 @@ Sequel.extension :eval_inspect
 
 module Sequel
   module SchemaDumper
+    # :nocov:
+    IGNORE_INDEX_ERRORS_KEY = RUBY_VERSION >= '3.4' ? 'ignore_index_errors: ' : ':ignore_index_errors=>'
+    # :nocov:
+    private_constant :IGNORE_INDEX_ERRORS_KEY
+
     # Convert the column schema information to a hash of column options, one of which must
     # be :type.  The other options added should modify that type (e.g. :size).  If a
     # database type is not recognized, return it as a String type.
@@ -161,7 +166,7 @@ END_MIG
     def dump_table_schema(table, options=OPTS)
       gen = dump_table_generator(table, options)
       commands = [gen.dump_columns, gen.dump_constraints, gen.dump_indexes].reject{|x| x == ''}.join("\n\n")
-      "create_table(#{table.inspect}#{', :ignore_index_errors=>true' if !options[:same_db] && options[:indexes] != false && !gen.indexes.empty?}) do\n#{commands.gsub(/^/, '  ')}\nend"
+      "create_table(#{table.inspect}#{", #{IGNORE_INDEX_ERRORS_KEY}true" if !options[:same_db] && options[:indexes] != false && !gen.indexes.empty?}) do\n#{commands.gsub(/^/, '  ')}\nend"
     end
 
     private
@@ -252,7 +257,7 @@ END_MIG
           gen.foreign_key(name, table, col_opts)
         else
           gen.column(name, type, col_opts)
-          if [Integer, :Bignum, Float, BigDecimal].include?(type) && schema[:db_type] =~ / unsigned\z/io
+          if [Integer, :Bignum, Float, BigDecimal].include?(type) && schema[:db_type] =~ / unsigned\z/i
             gen.check(Sequel::SQL::Identifier.new(name) >= 0)
           end
         end
@@ -425,6 +430,13 @@ END_MIG
 
   module Schema
     class CreateTableGenerator
+      # :nocov:
+      DEFAULT_KEY = RUBY_VERSION >= '3.4' ? 'default: ' : ':default=>'
+      IGNORE_ERRORS_KEY = RUBY_VERSION >= '3.4' ? 'ignore_errors: ' : ':ignore_errors=>'
+      # :nocov:
+      private_constant :DEFAULT_KEY
+      private_constant :IGNORE_ERRORS_KEY
+
       # Dump this generator's columns to a string that could be evaled inside
       # another instance to represent the same columns
       def dump_columns
@@ -508,7 +520,7 @@ END_MIG
           c = c.dup
           cols = c.delete(:columns)
           if table = options[:add_index] || options[:drop_index]
-            "#{options[:drop_index] ? 'drop' : 'add'}_index #{table.inspect}, #{cols.inspect}#{', :ignore_errors=>true' if options[:ignore_errors]}#{opts_inspect(c)}"
+            "#{options[:drop_index] ? 'drop' : 'add'}_index #{table.inspect}, #{cols.inspect}#{", #{IGNORE_ERRORS_KEY}true" if options[:ignore_errors]}#{opts_inspect(c)}"
           else
             "index #{cols.inspect}#{opts_inspect(c)}"
           end
@@ -526,7 +538,7 @@ END_MIG
         if opts[:default]
           opts = opts.dup
           de = Sequel.eval_inspect(opts.delete(:default)) 
-          ", :default=>#{de}#{", #{opts.inspect[1...-1]}" if opts.length > 0}"
+          ", #{DEFAULT_KEY}#{de}#{", #{opts.inspect[1...-1]}" if opts.length > 0}"
         else
           ", #{opts.inspect[1...-1]}" if opts.length > 0
         end

data/lib/sequel/extensions/sqlite_json_ops.rb

@@ -35,7 +35,7 @@
 #
 #   j[1]                     # (json_column ->> 1)
 #   j.get(1)                 # (json_column ->> 1)
-#   j.get_text(1)            # (json_column -> 1)
+#   j.get_json(1)            # (json_column -> 1)
 #   j.extract('$.a')         # json_extract(json_column, '$.a')
 #   jb.extract('$.a')        # jsonb_extract(jsonb_column, '$.a')
 #

data/lib/sequel/extensions/stdio_logger.rb

@@ -0,0 +1,48 @@
+# frozen-string-literal: true
+#
+# The stdio_logger extension exposes a Sequel::StdioLogger class that
+# can be used for logging with Sequel, as a minimal alternative to
+# the logger library.  It exposes debug/info/warn/error methods for the
+# different warning levels.  The debug method is a no-op, so that setting
+# the Database sql_log_level to debug will result in no output for normal
+# queries. The info/warn/error methods log the current time, log level,
+# and the given message.
+# 
+# To use this extension:
+#
+#   Sequel.extension :stdio_logger
+#
+# Then you you can use Sequel::StdioLogger to wrap IO objects that you
+# would like Sequel to log to:
+#
+#   DB.loggers << Sequel::StdioLogger.new($stdout)
+#
+#   log_file = File.open("db_queries.log", 'a')
+#   log_file.sync = true
+#   DB.loggers << Sequel::StdioLogger.new(log_file)
+#
+# This is implemented as a global extension instead of a Database extension
+# because Database loggers must be set before Database extensions are loaded.
+#
+# Related module: Sequel::StdioLogger
+
+#
+module Sequel
+  class StdioLogger
+    def initialize(device)
+      @device = device
+    end
+
+    # Do not log debug messages. This is so setting the Database
+    # sql_log_level to debug will result in no output.
+    def debug(msg)
+    end
+
+    [:info, :warn, :error].each do |meth|
+      define_method(meth) do |msg|
+        @device.write("#{Time.now.strftime('%F %T')} #{meth.to_s.upcase}: #{msg}\n")
+        nil
+      end
+    end
+  end
+end

data/lib/sequel/extensions/string_agg.rb

@@ -54,6 +54,7 @@
 # * MySQL
 # * HSQLDB
 # * H2
+# * SQLite 3.44+ (distinct only works when separator is ',')
 #
 # Related module: Sequel::SQL::StringAgg
 
@@ -94,6 +95,19 @@ module Sequel
           distinct = sa.is_distinct?
 
           case db_type = db.database_type
+          when :sqlite
+            raise Error, "string_agg(DISTINCT) is not supported with a non-comma separator on #{db.database_type}" if distinct && separator != ","
+
+            args = [expr]
+            args << separator unless distinct
+            f = Function.new(:group_concat, *args)
+            if order
+              f = f.order(*order)
+            end
+            if distinct
+              f = f.distinct
+            end
+            literal_append(sql, f)
           when :postgres, :sqlanywhere
             f = Function.new(db_type == :postgres ? :string_agg : :list, expr, separator)
             if order
@@ -151,7 +165,7 @@ module Sequel
         freeze
       end
 
-      # Whether the current expression uses distinct expressions 
+      # Whether the current expression uses distinct expressions
       def is_distinct?
         @distinct == true
       end
@@ -159,7 +173,7 @@ module Sequel
       # Return a modified StringAgg that uses distinct expressions
       def distinct
         self.class.new(@expr, @separator) do |sa|
-          sa.instance_variable_set(:@order_expr, @order_expr) if @order_expr
+          sa.instance_variable_set(:@order_expr, @order_expr)
           sa.instance_variable_set(:@distinct, true)
         end
       end
@@ -167,8 +181,8 @@ module Sequel
       # Return a modified StringAgg with the given order
       def order(*o)
         self.class.new(@expr, @separator) do |sa|
-          sa.instance_variable_set(:@distinct, @distinct) if @distinct
           sa.instance_variable_set(:@order_expr, o.empty? ? nil : o.freeze)
+          sa.instance_variable_set(:@distinct, @distinct)
         end
       end
 
@@ -178,4 +192,3 @@ module Sequel
 
   Dataset.register_extension(:string_agg, SQL::StringAgg::DatasetMethods)
 end
-