sequel 3.21.0 → 3.41.0

data/lib/sequel/extensions/pg_row_ops.rb

@@ -0,0 +1,182 @@
+# The pg_row_ops extension adds support to Sequel's DSL to make
+# it easier to deal with PostgreSQL row-valued/composite types.
+#
+# To load the extension:
+#
+#   Sequel.extension :pg_row_ops
+#
+# The most common usage is passing an expression to Sequel.pg_row_op:
+#
+#   r = Sequel.pg_row_op(:row_column)
+#
+# If you have also loaded the pg_row extension, you can use
+# Sequel.pg_row as well:
+#
+#   r = Sequel.pg_row(:row_column)
+#
+# Also, on most Sequel expression objects, you can call the pg_row
+# method:
+#
+#   r = Sequel.expr(:row_column).pg_row
+#
+# If you have loaded the {core_extensions extension}[link:files/doc/core_extensions_rdoc.html]),
+# you can also call Symbol#pg_row:
+#
+#   r = :row_column.pg_row
+#
+# There's only fairly basic support currently.  You can use the [] method to access
+# a member of the composite type:
+#
+#   r[:a] # (row_column).a
+#
+# This can be chained:
+#
+#   r[:a][:b] # ((row_column).a).b
+#
+# If you've loaded the pg_array_ops extension, you there is also support for composite
+# types that include arrays, or arrays of composite types:
+#
+#   r[1][:a] # (row_column[1]).a
+#   r[:a][1] # (row_column).a[1]
+#
+# The only other support is the splat method:
+#
+#   r.splat # (row_column.*)
+#
+# The splat method is necessary if you are trying to reference a table's type when the
+# table has the same name as one of it's columns.  For example:
+#
+#   DB.create_table(:a){Integer :a; Integer :b}
+#
+# Let's say you want to reference the composite type for the table:
+#
+#   a = Sequel.pg_row_op(:a)
+#   DB[:a].select(a[:b]) # SELECT (a).b FROM a
+#
+# Unfortunately, that doesn't work, as it references the integer column, not the table.
+# The splat method works around this:
+#
+#   DB[:a].select(a.splat[:b]) # SELECT (a.*).b FROM a
+#
+# Splat also takes an argument which is used for casting.  This is necessary if you
+# want to return the composite type itself, instead of the columns in the composite
+# type.  For example:
+#
+#   DB[:a].select(a.splat).first # SELECT (a.*) FROM a
+#   # => {:a=>1, :b=>2}
+#
+# By casting the expression, you can get a composite type returned:
+#
+#   DB[:a].select(a.splat).first # SELECT (a.*)::a FROM a
+#   # => {:a=>"(1,2)"} # or {:a=>{:a=>1, :b=>2}} if the "a" type has been registered
+#                      # with the the pg_row extension
+#
+# This feature is mostly useful for a different way to graph tables:
+#
+#   DB[:a].join(:b, :id=>:b_id).select(Sequel.pg_row_op(:a).splat(:a),
+#                                      Sequel.pg_row_op(:b).splat(:b))
+#   # SELECT (a.*)::a, (b.*)::b FROM a INNER JOIN b ON (b.id = a.b_id)
+#   # => {:a=>{:id=>1, :b_id=>2}, :b=>{:id=>2}}
+
+module Sequel
+  module Postgres
+    # This class represents a composite type expression reference.
+    class PGRowOp < SQL::PlaceholderLiteralString
+      OPEN = '('.freeze
+      CLOSE_DOT = ').'.freeze
+      CLOSE_STAR = '.*)'.freeze
+      CLOSE_STAR_CAST = '.*)::'.freeze
+      EMPTY = "".freeze
+      ROW = [OPEN, CLOSE_STAR].freeze
+      ROW_CAST = [OPEN, CLOSE_STAR_CAST].freeze
+      QUALIFY = [OPEN, CLOSE_DOT].freeze
+      WRAP = [EMPTY].freeze
+
+      # Wrap the expression in a PGRowOp, without changing the
+      # SQL it would use.
+      def self.wrap(expr)
+        PGRowOp.new(WRAP, [expr])
+      end
+
+      # Access a member of the composite type if given a
+      # symbol or an SQL::Identifier.  For all other access,
+      # assuming the pg_array_ops extension is loaded and
+      # that it represents an array access.  In either
+      # case, return a PgRowOp so that access can be cascaded.
+      def [](member)
+        case member
+        when Symbol, SQL::Identifier
+          PGRowOp.new(QUALIFY, [self, member])
+        else
+          PGRowOp.wrap(Sequel.pg_array_op(self)[member])
+        end
+      end
+
+      # Use the (identifier).* syntax to reference the members
+      # of the composite type as separate columns.  Generally
+      # used when you want to expand the columns of a composite
+      # type to be separate columns in the result set.
+      #
+      #   Sequel.pg_row_op(:a).*     # (a).*
+      #   Sequel.pg_row_op(:a)[:b].* # ((a).b).*
+      def *(ce=(arg=false;nil))
+        if arg == false
+          Sequel::SQL::ColumnAll.new([self])
+        else
+          super(ce)
+        end
+      end
+
+      # Use the (identifier.*) syntax to indicate that this
+      # expression represents the composite type of one
+      # of the tables being referenced, if it has the same
+      # name as one of the columns.  If the cast_to argument
+      # is given, also cast the expression to that type
+      # (which should be a symbol representing the composite type).
+      # This is used if you want to return whole table row as a
+      # composite type.
+      #
+      #   Sequel.pg_row_op(:a).splat[:b] # (a.*).b
+      #   Sequel.pg_row_op(:a).splat(:a) # (a.*)::a
+      def splat(cast_to=nil)
+        if args.length > 1
+          raise Error, 'cannot splat a PGRowOp with multiple arguments'
+        end
+
+        if cast_to
+          PGRowOp.new(ROW_CAST, args + [cast_to])
+        else
+          PGRowOp.new(ROW, args)
+        end
+      end
+
+      module ExpressionMethods
+        # Return a PGRowOp wrapping the receiver.
+        def pg_row
+          Sequel.pg_row_op(self)
+        end
+      end
+    end
+  end
+
+  module SQL::Builders
+    # Return a PGRowOp wrapping the given expression.
+    def pg_row_op(expr)
+      Postgres::PGRowOp.wrap(expr)
+    end
+  end
+
+  class SQL::GenericExpression
+    include Sequel::Postgres::PGRowOp::ExpressionMethods
+  end
+
+  class LiteralString
+    include Sequel::Postgres::PGRowOp::ExpressionMethods
+  end
+end
+
+if Sequel.core_extensions?
+  class Symbol
+    include Sequel::Postgres::PGRowOp::ExpressionMethods
+  end
+end

data/lib/sequel/extensions/pg_statement_cache.rb

@@ -0,0 +1,317 @@
+# This extension adds a statement cache to Sequel's postgres adapter,
+# with the ability to automatically prepare statements that are
+# executed repeatedly.  When combined with the pg_auto_parameterize
+# extension, it can take Sequel code such as:
+#
+#   DB.extension :pg_auto_parameterize, :pg_statement_cache
+#   DB[:table].filter(:a=>1)
+#   DB[:table].filter(:a=>2)
+#   DB[:table].filter(:a=>3)
+#
+# And use the same prepared statement to execute the queries.
+#
+# The backbone of this extension is a modified LRU cache.  It considers
+# both the last executed time and the number of executions when
+# determining which queries to keep in the cache.  It only cleans the
+# cache when a high water mark has been passed, and removes queries
+# until it reaches the low water mark, in order to avoid thrashing when
+# you are using more than the maximum number of queries.  To avoid
+# preparing queries when it isn't necessary, it does not prepare them
+# on the server side unless they are being executed more than once.
+# The cache is very tunable, allowing you to set the high and low
+# water marks, the number of executions before preparing the query,
+# and even use a custom callback for determine which queries to keep
+# in the cache.
+#
+# Note that automatically preparing statements does have some issues.
+# Most notably, if you change the result type that the query returns,
+# PostgreSQL will raise an error.  This can happen if you have
+# prepared a statement that selects all columns from a table, and then
+# you add or remove a column from that table.  This extension does
+# attempt to check that case and clear the statement caches if you use
+# alter_table from within Sequel, but it cannot fix the case when such
+# a change is made externally.
+#
+# This extension only works when the pg driver is used as the backend
+# for the postgres adapter.
+
+module Sequel
+  module Postgres
+    module StatementCache
+      # A simple structure used for the values in the StatementCache's hash.
+      # It does not hold the related SQL, since that is used as the key for
+      # the StatementCache's hash.
+      class Statement
+        # The last time this statement was seen by the cache, persumably the
+        # last time it was executed.
+        attr_accessor :last_seen
+
+        # The total number of executions since the statement entered the cache.
+        attr_accessor :num_executes
+
+        # The id related to the statement, used as part of the prepared statement
+        # name.
+        attr_reader :cache_id
+
+        # Used when adding entries to the cache, just sets their id.  Uses
+        # 0 for num_executes since that is incremented elsewhere. Does not
+        # set last_seen since that is set elsewhere to reduce branching.
+        def initialize(cache_id)
+          @num_executes = 0
+          @cache_id = cache_id
+        end
+
+        # The name to use for the server side prepared statement.  Note that this
+        # statement might not actually be prepared.
+        def name
+          "sequel_pgap_#{cache_id}"
+        end
+      end
+
+      # The backbone of the block, a modified LRU (least recently used) cache
+      # mapping SQL query strings to Statement objects.
+      class StatementCache
+        include Enumerable
+
+        # Set the options for the statement cache.  These are generally set at
+        # the database level using the :statement_cache_opts Database option.
+        #
+        # :max_size :: The maximum size (high water mark) for the cache.  If
+        #              an entry is added when the current size of the cache is
+        #              equal to the maximum size, the cache is cleaned up to
+        #              reduce the number of entries to the :min_size.  Defaults
+        #              to 1000.
+        # :min_size :: The minimum size (low water mark) for the cache.  On
+        #              cleanup, the size of the cache is reduced to this
+        #              number.  Note that there could be fewer than this
+        #              number of entries in the cache.  Defaults to :max_size/2. 
+        # :prepare_after :: The number of executions to wait for before preparing
+        #              the query server-side.  If set to 1, prepares all executed
+        #              queries server-side.  If set to 5, does not attempt to
+        #              prepare the query until the 5th execution.  Defaults to 2.
+        # :sorter :: A callable object that takes two arguments, the current time
+        #            and the related Statement instance, and should return some
+        #            Comparable (usually a numeric) such that the lowest values
+        #            returned are the first to be removed when it comes time to
+        #            clean the pool.  The default is basically:
+        #             
+        #              lambda{|t, stmt| (stmt.last_seen - t)/stmt.num_executes}
+        #
+        #            so that it doesn't remove statements that have been executed
+        #            many times just because many less-frequently executed statements 
+        #            have been executed recently.
+        #
+        # The block passed is called with the Statement object's name, only for
+        # statements that have been prepared, and should be used to deallocate the
+        # statements.
+        def initialize(opts={}, &block)
+          @cleanup_proc = block
+          @prepare_after = opts.fetch(:prepare_after, 2)
+          @max_size = opts.fetch(:max_size, 1000)
+          @min_size = opts.fetch(:min_size, @max_size/2)
+          @sorter = opts.fetch(:sorter){method(:default_sorter)}
+          @ids = (1..@max_size).to_a.reverse
+          @hash = {}
+          #
+          # We add one so that when we clean the cache, the entry
+          # about to be added brings us to the min_size.
+          @size_diff = @max_size - @min_size + 1
+        end
+        
+        # Completely clear the statement cache, deallocating on
+        # the server side all statements that have been prepared.
+        def clear
+          @hash.keys.each{|k| remove(k)}
+        end
+
+        # Yield each SQL string and Statement instance in the cache
+        # to the block.
+        def each(&block)
+          @hash.each(&block)
+        end
+
+        # Get the related statement name from the cache.  If the
+        # entry is already in the cache, just bump it's last seen
+        # time and the number of executions.  Otherwise, add it
+        # to the cache.  If the cache is already full, clean it up
+        # before adding it.
+        #
+        # If the num of executions has passed the threshhold, yield
+        # the statement name to the block, which should be used to
+        # prepare the statement on the server side.
+        #
+        # This method should return the prepared statment name if
+        # the statement has been prepared, and nil if the query
+        # has not been prepared and the statement should be executed
+        # normally.
+        def fetch(sql)
+          unless stmt = @hash[sql]
+            # Get the next id from the id pool.
+            unless id = @ids.pop
+              # No id left, cache must be full, so cleanup and then
+              # get the next id from the id pool.
+              cleanup
+              id = @ids.pop
+            end
+            @hash[sql] = stmt = Statement.new(id)
+          end
+
+          stmt.last_seen = Time.now
+          stmt.num_executes += 1
+
+          if stmt.num_executes >= @prepare_after
+            if stmt.num_executes == @prepare_after
+              begin
+                yield(stmt.name)
+              rescue PGError
+                # An error occurred while preparing the statement,
+                # execute it normally (which will probably raise
+                # the error again elsewhere), but decrement the
+                # number of executions so we don't think we've
+                # prepared the statement when we haven't.
+                stmt.num_executes -= 1
+                return nil
+              end
+            end
+            stmt.name
+          end
+        end
+
+        # The current size of the statement cache.
+        def size
+          @hash.length
+        end
+
+        private
+
+        # Sort by time since last execution and number of executions.
+        # We don't want to throw stuff out of the
+        # cache if it has been executed a lot,
+        # but a bunch of queries that were
+        # executed only once came in more recently.
+        def default_sorter(t, stmt)
+          (stmt.last_seen - t)/stmt.num_executes
+        end
+
+        # After sorting the cache appropriately (so that the least important
+        # items are first), reduce the number of entries in the cache to
+        # the low water mark by removing the related query.  Should only be
+        # called when the cache is full.
+        def cleanup
+          t = Time.now
+          @hash.sort_by{|k,v| @sorter.call(t, v)}.first(@size_diff).each{|sql, stmt| remove(sql)}
+        end
+
+        # Remove the query from the cache.  If it has been prepared,
+        # call the cleanup_proc to deallocate the statement.
+        def remove(sql)
+          stmt = @hash.delete(sql)
+          if stmt.num_executes >= @prepare_after
+            @cleanup_proc.call(stmt.name)
+          end
+
+          # Return id to the pool of ids
+          @ids.push(stmt.cache_id)
+        end
+      end
+
+      module AdapterMethods
+        # A regular expression for the types of queries to cache.  Any queries not
+        # matching this regular expression are not cached.
+        DML_RE = /\A(WITH|SELECT|INSERT|UPDATE|DELETE) /
+
+        # The StatementCache instance for this connection.  Note that
+        # each connection has a separate StatementCache, because prepared
+        # statements are connection-specific.
+        attr_reader :statement_cache
+
+        # Set the statement_cache for the connection, using the database's
+        # :statement_cache_opts option.
+        def self.extended(c)
+          c.instance_variable_set(:@statement_cache, StatementCache.new(c.sequel_db.opts[:statement_cache_opts] || {}){|name| c.deallocate(name)})
+        end
+
+        # pg seems to already use the db method (but not the @db instance variable),
+        # so use the sequel_db method to access the related Sequel::Database object.
+        def sequel_db
+          @db
+        end
+
+        # Deallocate on the server the prepared statement with the given name.
+        def deallocate(name)
+          begin
+            execute("DEALLOCATE #{name}")
+          rescue PGError
+            # table probably got removed, just ignore it
+          end
+        end
+
+        private
+
+        # If the sql query string is one we should cache, cache it.  If the query already
+        # has a related prepared statement with it, execute the prepared statement instead
+        # of executing the query normally.
+        def execute_query(sql, args=nil)
+          if sql =~ DML_RE
+            if name = statement_cache.fetch(sql){|stmt_name| sequel_db.log_yield("PREPARE #{stmt_name} AS #{sql}"){prepare(stmt_name, sql)}}
+              if args
+                sequel_db.log_yield("EXECUTE #{name} (#{sql})", args){exec_prepared(name, args)}
+              else
+                sequel_db.log_yield("EXECUTE #{name} (#{sql})"){exec_prepared(name)}
+              end
+            else
+              super
+            end
+          else
+            super
+          end
+        end
+      end
+      
+      module DatabaseMethods
+        # Setup the after_connect proc for the connection pool to make
+        # sure the connection object is extended with the appropriate
+        # module.  This disconnects any existing connections to ensure
+        # that all connections in the pool have been extended appropriately.
+        def self.extended(db)
+          # Respect existing after_connect proc if one is present
+          pr = db.opts[:after_connect]
+
+          # Set the after_connect proc to extend the adapter with
+          # the statement cache support.
+          db.pool.after_connect = db.opts[:after_connect] = proc do |c|
+            pr.call(c) if pr
+            c.extend(AdapterMethods)
+          end
+
+          # Disconnect to make sure all connections get set up with
+          # statement cache.
+          db.disconnect
+        end
+
+        # Clear statement caches for all connections when altering tables.
+        def alter_table(*)
+          clear_statement_caches
+          super
+        end
+
+        # Clear statement caches for all connections when dropping tables.
+        def drop_table(*)
+          clear_statement_caches
+          super
+        end
+
+        private
+
+        # Clear the statement cache for all connections.  Note that for
+        # the threaded pools, this will not affect connections currently
+        # allocated to other threads.
+        def clear_statement_caches
+          pool.all_connections{|c| c.statement_cache.clear}
+        end
+      end
+    end
+  end
+
+  Database.register_extension(:pg_statement_cache, Postgres::StatementCache::DatabaseMethods)
+end

data/lib/sequel/extensions/pretty_table.rb

@@ -1,82 +1,27 @@
 # The pretty_table extension adds Sequel::Dataset#print and the
 # Sequel::PrettyTable class for creating nice-looking plain-text
-# tables.
+# tables.  Example:
+#
+#   +--+-------+
+#   |id|name   |
+#   |--+-------|
+#   |1 |fasdfas|
+#   |2 |test   |
+#   +--+-------+
+#
+# To load the extension:
+#
+#   Sequel.extension :pretty_table
 
 module Sequel
+  extension :_pretty_table
+
   class Dataset
     # Pretty prints the records in the dataset as plain-text table.
     def print(*cols)
-      Sequel::PrettyTable.print(naked.all, cols.empty? ? columns : cols)
+      ds = naked
+      rows = ds.all
+      Sequel::PrettyTable.print(rows, cols.empty? ? ds.columns : cols)
     end
   end
-
-  module PrettyTable
-    # Prints nice-looking plain-text tables via puts
-    # 
-    #   +--+-------+
-    #   |id|name   |
-    #   |--+-------|
-    #   |1 |fasdfas|
-    #   |2 |test   |
-    #   +--+-------+
-    def self.print(records, columns = nil) # records is an array of hashes
-      columns ||= records.first.keys.sort_by{|x|x.to_s}
-      sizes = column_sizes(records, columns)
-      sep_line = separator_line(columns, sizes)
-
-      puts sep_line
-      puts header_line(columns, sizes)
-      puts sep_line
-      records.each {|r| puts data_line(columns, sizes, r)}
-      puts sep_line
-    end
-
-    ### Private Module Methods ###
-
-    # Hash of the maximum size of the value for each column 
-    def self.column_sizes(records, columns) # :nodoc:
-      sizes = Hash.new {0}
-      columns.each do |c|
-        s = c.to_s.size
-        sizes[c.to_sym] = s if s > sizes[c.to_sym]
-      end
-      records.each do |r|
-        columns.each do |c|
-          s = r[c].to_s.size
-          sizes[c.to_sym] = s if s > sizes[c.to_sym]
-        end
-      end
-      sizes
-    end
-    
-    # String for each data line
-    def self.data_line(columns, sizes, record) # :nodoc:
-      '|' << columns.map {|c| format_cell(sizes[c], record[c])}.join('|') << '|'
-    end
-    
-    # Format the value so it takes up exactly size characters
-    def self.format_cell(size, v) # :nodoc:
-      case v
-      when Bignum, Fixnum
-        "%#{size}d" % v
-      when Float
-        "%#{size}g" % v
-      else
-        "%-#{size}s" % v.to_s
-      end
-    end
-    
-    # String for header line
-    def self.header_line(columns, sizes) # :nodoc:
-      '|' << columns.map {|c| "%-#{sizes[c]}s" % c.to_s}.join('|') << '|'
-    end
-
-    # String for separtor line
-    def self.separator_line(columns, sizes) # :nodoc:
-      '+' << columns.map {|c| '-' * sizes[c]}.join('+') << '+'
-    end
-
-    private_class_method :column_sizes, :data_line, :format_cell, :header_line, :separator_line
-  end
 end
-

data/lib/sequel/extensions/query.rb

@@ -1,6 +1,10 @@
 # The query extension adds Sequel::Dataset#query which allows
 # a different way to construct queries instead of the usual
 # method chaining.
+#
+# To load the extension, do:
+#
+#   Sequel.extension :query
 
 module Sequel
   class Database
@@ -16,13 +20,13 @@ module Sequel
     #
     #   dataset = DB[:items].query do
     #     select :x, :y, :z
-    #     filter{|o| (o.x > 1) & (o.y > 2)}
-    #     order :z.desc
+    #     filter{(x > 1) & (y > 2)}
+    #     reverse :z
     #   end
     #
     # Which is the same as:
     #
-    #  dataset = DB[:items].select(:x, :y, :z).filter{|o| (o.x > 1) & (o.y > 2)}.order(:z.desc)
+    #  dataset = DB[:items].select(:x, :y, :z).filter{(x > 1) & (y > 2)}.reverse(:z)
     #
     # Note that inside a call to query, you cannot call each, insert, update,
     # or delete (or any method that calls those), or Sequel will raise an
@@ -43,7 +47,7 @@ module Sequel
 
       # Merge the given options into the receiver's options and return the receiver
       # instead of cloning the receiver.
-      def clone(opts = nil)
+      def clone(opts = {})
         @opts.merge!(opts)
         self
       end