sequel 3.21.0 → 3.36.0

data/lib/sequel/extensions/pg_statement_cache.rb

@@ -0,0 +1,316 @@
+# This extension adds a statement cache to Sequel's postgres adapter,
+# with the ability to automatically prepare statements that are
+# executed repeatedly.  When combined with the pg_auto_parameterize
+# extension, it can take Sequel code such as:
+#
+#   DB.extend Sequel::Postgres::AutoParameterize::DatabaseMethods
+#   DB.extend Sequel::Postgres::StatementCache::DatabaseMethods
+#   DB[:table].filter(:a=>1)
+#   DB[:table].filter(:a=>2)
+#   DB[:table].filter(:a=>3)
+#
+# And use the same prepared statement to execute the queries.
+#
+# The backbone of this extension is a modified LRU cache.  It considers
+# both the last executed time and the number of executions when
+# determining which queries to keep in the cache.  It only cleans the
+# cache when a high water mark has been passed, and removes queries
+# until it reaches the low water mark, in order to avoid thrashing when
+# you are using more than the maximum number of queries.  To avoid
+# preparing queries when it isn't necessary, it does not prepare them
+# on the server side unless they are being executed more than once.
+# The cache is very tunable, allowing you to set the high and low
+# water marks, the number of executions before preparing the query,
+# and even use a custom callback for determine which queries to keep
+# in the cache.
+#
+# Note that automatically preparing statements does have some issues.
+# Most notably, if you change the result type that the query returns,
+# PostgreSQL will raise an error.  This can happen if you have
+# prepared a statement that selects all columns from a table, and then
+# you add or remove a column from that table.  This extension does
+# attempt to check that case and clear the statement caches if you use
+# alter_table from within Sequel, but it cannot fix the case when such
+# a change is made externally.
+#
+# This extension only works when the pg driver is used as the backend
+# for the postgres adapter.
+
+module Sequel
+  module Postgres
+    module StatementCache
+      # A simple structure used for the values in the StatementCache's hash.
+      # It does not hold the related SQL, since that is used as the key for
+      # the StatementCache's hash.
+      class Statement
+        # The last time this statement was seen by the cache, persumably the
+        # last time it was executed.
+        attr_accessor :last_seen
+
+        # The total number of executions since the statement entered the cache.
+        attr_accessor :num_executes
+
+        # The id related to the statement, used as part of the prepared statement
+        # name.
+        attr_reader :cache_id
+
+        # Used when adding entries to the cache, just sets their id.  Uses
+        # 0 for num_executes since that is incremented elsewhere. Does not
+        # set last_seen since that is set elsewhere to reduce branching.
+        def initialize(cache_id)
+          @num_executes = 0
+          @cache_id = cache_id
+        end
+
+        # The name to use for the server side prepared statement.  Note that this
+        # statement might not actually be prepared.
+        def name
+          "sequel_pgap_#{cache_id}"
+        end
+      end
+
+      # The backbone of the block, a modified LRU (least recently used) cache
+      # mapping SQL query strings to Statement objects.
+      class StatementCache
+        include Enumerable
+
+        # Set the options for the statement cache.  These are generally set at
+        # the database level using the :statement_cache_opts Database option.
+        #
+        # :max_size :: The maximum size (high water mark) for the cache.  If
+        #              an entry is added when the current size of the cache is
+        #              equal to the maximum size, the cache is cleaned up to
+        #              reduce the number of entries to the :min_size.  Defaults
+        #              to 1000.
+        # :min_size :: The minimum size (low water mark) for the cache.  On
+        #              cleanup, the size of the cache is reduced to this
+        #              number.  Note that there could be fewer than this
+        #              number of entries in the cache.  Defaults to :max_size/2. 
+        # :prepare_after :: The number of executions to wait for before preparing
+        #              the query server-side.  If set to 1, prepares all executed
+        #              queries server-side.  If set to 5, does not attempt to
+        #              prepare the query until the 5th execution.  Defaults to 2.
+        # :sorter :: A callable object that takes two arguments, the current time
+        #            and the related Statement instance, and should return some
+        #            Comparable (usually a numeric) such that the lowest values
+        #            returned are the first to be removed when it comes time to
+        #            clean the pool.  The default is basically:
+        #             
+        #              lambda{|t, stmt| (stmt.last_seen - t)/stmt.num_executes}
+        #
+        #            so that it doesn't remove statements that have been executed
+        #            many times just because many less-frequently executed statements 
+        #            have been executed recently.
+        #
+        # The block passed is called with the Statement object's name, only for
+        # statements that have been prepared, and should be used to deallocate the
+        # statements.
+        def initialize(opts={}, &block)
+          @cleanup_proc = block
+          @prepare_after = opts.fetch(:prepare_after, 2)
+          @max_size = opts.fetch(:max_size, 1000)
+          @min_size = opts.fetch(:min_size, @max_size/2)
+          @sorter = opts.fetch(:sorter){method(:default_sorter)}
+          @ids = (1..@max_size).to_a.reverse
+          @hash = {}
+          #
+          # We add one so that when we clean the cache, the entry
+          # about to be added brings us to the min_size.
+          @size_diff = @max_size - @min_size + 1
+        end
+        
+        # Completely clear the statement cache, deallocating on
+        # the server side all statements that have been prepared.
+        def clear
+          @hash.keys.each{|k| remove(k)}
+        end
+
+        # Yield each SQL string and Statement instance in the cache
+        # to the block.
+        def each(&block)
+          @hash.each(&block)
+        end
+
+        # Get the related statement name from the cache.  If the
+        # entry is already in the cache, just bump it's last seen
+        # time and the number of executions.  Otherwise, add it
+        # to the cache.  If the cache is already full, clean it up
+        # before adding it.
+        #
+        # If the num of executions has passed the threshhold, yield
+        # the statement name to the block, which should be used to
+        # prepare the statement on the server side.
+        #
+        # This method should return the prepared statment name if
+        # the statement has been prepared, and nil if the query
+        # has not been prepared and the statement should be executed
+        # normally.
+        def fetch(sql)
+          unless stmt = @hash[sql]
+            # Get the next id from the id pool.
+            unless id = @ids.pop
+              # No id left, cache must be full, so cleanup and then
+              # get the next id from the id pool.
+              cleanup
+              id = @ids.pop
+            end
+            @hash[sql] = stmt = Statement.new(id)
+          end
+
+          stmt.last_seen = Time.now
+          stmt.num_executes += 1
+
+          if stmt.num_executes >= @prepare_after
+            if stmt.num_executes == @prepare_after
+              begin
+                yield(stmt.name)
+              rescue PGError
+                # An error occurred while preparing the statement,
+                # execute it normally (which will probably raise
+                # the error again elsewhere), but decrement the
+                # number of executions so we don't think we've
+                # prepared the statement when we haven't.
+                stmt.num_executes -= 1
+                return nil
+              end
+            end
+            stmt.name
+          end
+        end
+
+        # The current size of the statement cache.
+        def size
+          @hash.length
+        end
+
+        private
+
+        # Sort by time since last execution and number of executions.
+        # We don't want to throw stuff out of the
+        # cache if it has been executed a lot,
+        # but a bunch of queries that were
+        # executed only once came in more recently.
+        def default_sorter(t, stmt)
+          (stmt.last_seen - t)/stmt.num_executes
+        end
+
+        # After sorting the cache appropriately (so that the least important
+        # items are first), reduce the number of entries in the cache to
+        # the low water mark by removing the related query.  Should only be
+        # called when the cache is full.
+        def cleanup
+          t = Time.now
+          @hash.sort_by{|k,v| @sorter.call(t, v)}.first(@size_diff).each{|sql, stmt| remove(sql)}
+        end
+
+        # Remove the query from the cache.  If it has been prepared,
+        # call the cleanup_proc to deallocate the statement.
+        def remove(sql)
+          stmt = @hash.delete(sql)
+          if stmt.num_executes >= @prepare_after
+            @cleanup_proc.call(stmt.name)
+          end
+
+          # Return id to the pool of ids
+          @ids.push(stmt.cache_id)
+        end
+      end
+
+      module AdapterMethods
+        # A regular expression for the types of queries to cache.  Any queries not
+        # matching this regular expression are not cached.
+        DML_RE = /\A(WITH|SELECT|INSERT|UPDATE|DELETE) /
+
+        # The StatementCache instance for this connection.  Note that
+        # each connection has a separate StatementCache, because prepared
+        # statements are connection-specific.
+        attr_reader :statement_cache
+
+        # Set the statement_cache for the connection, using the database's
+        # :statement_cache_opts option.
+        def self.extended(c)
+          c.instance_variable_set(:@statement_cache, StatementCache.new(c.sequel_db.opts[:statement_cache_opts] || {}){|name| c.deallocate(name)})
+        end
+
+        # pg seems to already use the db method (but not the @db instance variable),
+        # so use the sequel_db method to access the related Sequel::Database object.
+        def sequel_db
+          @db
+        end
+
+        # Deallocate on the server the prepared statement with the given name.
+        def deallocate(name)
+          begin
+            execute("DEALLOCATE #{name}")
+          rescue PGError
+            # table probably got removed, just ignore it
+          end
+        end
+
+        private
+
+        # If the sql query string is one we should cache, cache it.  If the query already
+        # has a related prepared statement with it, execute the prepared statement instead
+        # of executing the query normally.
+        def execute_query(sql, args=nil)
+          if sql =~ DML_RE
+            if name = statement_cache.fetch(sql){|stmt_name| sequel_db.log_yield("PREPARE #{stmt_name} AS #{sql}"){prepare(stmt_name, sql)}}
+              if args
+                sequel_db.log_yield("EXECUTE #{name} (#{sql})", args){exec_prepared(name, args)}
+              else
+                sequel_db.log_yield("EXECUTE #{name} (#{sql})"){exec_prepared(name)}
+              end
+            else
+              super
+            end
+          else
+            super
+          end
+        end
+      end
+      
+      module DatabaseMethods
+        # Setup the after_connect proc for the connection pool to make
+        # sure the connection object is extended with the appropriate
+        # module.  This disconnects any existing connections to ensure
+        # that all connections in the pool have been extended appropriately.
+        def self.extended(db)
+          # Respect existing after_connect proc if one is present
+          pr = db.opts[:after_connect]
+
+          # Set the after_connect proc to extend the adapter with
+          # the statement cache support.
+          db.pool.after_connect = db.opts[:after_connect] = proc do |c|
+            pr.call(c) if pr
+            c.extend(AdapterMethods)
+          end
+
+          # Disconnect to make sure all connections get set up with
+          # statement cache.
+          db.disconnect
+        end
+
+        # Clear statement caches for all connections when altering tables.
+        def alter_table(*)
+          clear_statement_caches
+          super
+        end
+
+        # Clear statement caches for all connections when dropping tables.
+        def drop_table(*)
+          clear_statement_caches
+          super
+        end
+
+        private
+
+        # Clear the statement cache for all connections.  Note that for
+        # the threaded pools, this will not affect connections currently
+        # allocated to other threads.
+        def clear_statement_caches
+          pool.all_connections{|c| c.statement_cache.clear}
+        end
+      end
+    end
+  end
+end

data/lib/sequel/extensions/pretty_table.rb

@@ -3,80 +3,14 @@
 # tables.
 
 module Sequel
+  extension :_pretty_table
+
   class Dataset
     # Pretty prints the records in the dataset as plain-text table.
     def print(*cols)
-      Sequel::PrettyTable.print(naked.all, cols.empty? ? columns : cols)
+      ds = naked
+      rows = ds.all
+      Sequel::PrettyTable.print(rows, cols.empty? ? ds.columns : cols)
     end
   end
-
-  module PrettyTable
-    # Prints nice-looking plain-text tables via puts
-    # 
-    #   +--+-------+
-    #   |id|name   |
-    #   |--+-------|
-    #   |1 |fasdfas|
-    #   |2 |test   |
-    #   +--+-------+
-    def self.print(records, columns = nil) # records is an array of hashes
-      columns ||= records.first.keys.sort_by{|x|x.to_s}
-      sizes = column_sizes(records, columns)
-      sep_line = separator_line(columns, sizes)
-
-      puts sep_line
-      puts header_line(columns, sizes)
-      puts sep_line
-      records.each {|r| puts data_line(columns, sizes, r)}
-      puts sep_line
-    end
-
-    ### Private Module Methods ###
-
-    # Hash of the maximum size of the value for each column 
-    def self.column_sizes(records, columns) # :nodoc:
-      sizes = Hash.new {0}
-      columns.each do |c|
-        s = c.to_s.size
-        sizes[c.to_sym] = s if s > sizes[c.to_sym]
-      end
-      records.each do |r|
-        columns.each do |c|
-          s = r[c].to_s.size
-          sizes[c.to_sym] = s if s > sizes[c.to_sym]
-        end
-      end
-      sizes
-    end
-    
-    # String for each data line
-    def self.data_line(columns, sizes, record) # :nodoc:
-      '|' << columns.map {|c| format_cell(sizes[c], record[c])}.join('|') << '|'
-    end
-    
-    # Format the value so it takes up exactly size characters
-    def self.format_cell(size, v) # :nodoc:
-      case v
-      when Bignum, Fixnum
-        "%#{size}d" % v
-      when Float
-        "%#{size}g" % v
-      else
-        "%-#{size}s" % v.to_s
-      end
-    end
-    
-    # String for header line
-    def self.header_line(columns, sizes) # :nodoc:
-      '|' << columns.map {|c| "%-#{sizes[c]}s" % c.to_s}.join('|') << '|'
-    end
-
-    # String for separtor line
-    def self.separator_line(columns, sizes) # :nodoc:
-      '+' << columns.map {|c| '-' * sizes[c]}.join('+') << '+'
-    end
-
-    private_class_method :column_sizes, :data_line, :format_cell, :header_line, :separator_line
-  end
 end
-

data/lib/sequel/extensions/query.rb

@@ -43,7 +43,7 @@ module Sequel
 
       # Merge the given options into the receiver's options and return the receiver
       # instead of cloning the receiver.
-      def clone(opts = nil)
+      def clone(opts = {})
         @opts.merge!(opts)
         self
       end

data/lib/sequel/extensions/query_literals.rb

@@ -0,0 +1,79 @@
+# The query_literals extension changes Sequel's default behavior of
+# the select, order and group methods so that if the first argument
+# is a regular string, it is treated as a literal string, with the
+# rest of the arguments (if any) treated as placeholder values. This
+# allows you to write code such as:
+#
+#   DB[:table].select('a, b, ?', 2).group('a, b').order('c')
+#
+# The default Sequel behavior would literalize that as:
+#
+#   SELECT 'a, b, ?', 2 FROM table GROUP BY 'a, b' ORDER BY 'c'
+#
+# Using this extension changes the literalization to:
+#
+#   SELECT a, b, 2, FROM table GROUP BY a, b ORDER BY c
+#
+# This extension makes select, group, and order methods operate
+# like filter methods, which support the same interface.
+#
+# There are very few places where Sequel's default behavior is
+# desirable in this area, but for backwards compatibility, the
+# defaults won't be changed until the next major release.
+#
+# Loading this extension does nothing by default except make the
+# Sequel::QueryLiterals module available.  You can extend specific
+# datasets with this module:
+#
+#   ds = DB[:table]
+#   ds.extend(Sequel::QueryLiterals)
+#
+# Order you can extend all of a database's datasets with it, which
+# is probably the desired behavior if you are using this extension:
+#
+#   DB.extend_datasets(Sequel::QueryLiterals)
+
+module Sequel
+  # The QueryLiterals module can be used to make select, group, and
+  # order methods operate similar to the filter methods if the first
+  # argument is a plain string, treating it like a literal string,
+  # with any remaining arguments treated as placeholder values.
+  #
+  # This adds such support to the following methods: select, select_append,
+  # select_group, select_more, group, group_and_count, order, order_append,
+  # and order_more.
+  #
+  # Note that if you pass a block to these methods, it will use the default
+  # implementation without the special literal handling.
+  module QueryLiterals
+    %w'select select_append select_group select_more group group_and_count order order_append order_more'.each do |m|
+      class_eval(<<-END, __FILE__, __LINE__ + 1)
+        def #{m}(*args)
+          if !block_given? && (l = query_literal(args))
+            super(l)
+          else
+            super
+          end
+        end
+      END
+    end
+
+    private
+
+    # If the first argument is a plain string, return a literal string
+    # if there are no additional args or a placeholder literal string with
+    # the remaining args.  Otherwise, return nil.
+    def query_literal(args)
+      case (s = args[0])
+      when LiteralString, SQL::Blob
+        nil
+      when String
+        if args.length == 1
+          LiteralString.new(s)
+        else
+          SQL::PlaceholderLiteralString.new(s, args[1..-1])
+        end
+      end
+    end
+  end
+end

data/lib/sequel/extensions/schema_caching.rb

@@ -0,0 +1,76 @@
+# The schema_caching extension adds a few methods to Sequel::Database
+# that make it easy to dump the parsed schema information to a file,
+# and load it from that file.  Loading the schema information from a
+# dumped file is faster than parsing it from the database, so this
+# can save bootup time for applications with large numbers of models.
+#
+# Basic usage in application code:
+#
+#   Sequel.extension :schema_caching
+#
+#   DB = Sequel.connect('...')
+#
+#   DB.load_schema_cache('/path/to/schema.dump')
+#
+#   # load model files
+#
+# Then, whenever the database schema is modified, write a new cached
+# file.  You can do that with <tt>bin/sequel</tt>'s -S option:
+#
+#   bin/sequel -S /path/to/schema.dump postgres://...
+#
+# Alternatively, if you don't want to dump the schema information for
+# all tables, and you don't worry about race conditions, you can
+# choose to use the following in your application code:
+#
+#   Sequel.extension :schema_caching
+#
+#   DB = Sequel.connect('...')
+#
+#   DB.load_schema_cache?('/path/to/schema.dump')
+#
+#   # load model files
+#
+#   DB.dump_schema_cache?('/path/to/schema.dump')
+#
+# With this method, you just have to delete the schema dump file if
+# the schema is modified, and the application will recreate it for you
+# using just the tables that your models use.
+#
+# Note that it is up to the application to ensure that the dumped
+# cached schema reflects the current state of the database.  Sequel
+# does no checking to ensure this, as checking would take time and the
+# purpose of this code is to take a shortcut.
+#
+# The cached schema is dumped in Marshal format, since it is the fastest
+# and it handles all ruby objects used in the schema hash.  Because of this,
+# you should not attempt to load the schema from a untrusted file.
+
+module Sequel
+  class Database
+    # Dump the cached schema to the filename given in Marshal format.
+    def dump_schema_cache(file)
+      File.open(file, 'wb'){|f| f.write(Marshal.dump(@schemas))}
+      nil
+    end
+
+    # Dump the cached schema to the filename given unless the file
+    # already exists.
+    def dump_schema_cache?(file)
+      dump_schema_cache(file) unless File.exist?(file)
+    end
+
+    # Replace the schema cache with the data from the given file, which
+    # should be in Marshal format.
+    def load_schema_cache(file)
+      @schemas = Marshal.load(File.read(file))
+      nil
+    end
+
+    # Replace the schema cache with the data from the given file if the
+    # file exists.
+    def load_schema_cache?(file)
+      load_schema_cache(file) if File.exist?(file)
+    end
+  end
+end