sequel 3.3.0 → 3.4.0

data/lib/sequel/adapters/sqlite.rb

@@ -32,10 +32,10 @@ module Sequel
         db.busy_timeout(opts.fetch(:timeout, 5000))
         db.type_translation = true
         
-        # Handle datetime's with Sequel.datetime_class
+        # Handle datetimes with Sequel.datetime_class
         prok = proc do |t,v|
           v = Time.at(v.to_i).iso8601 if UNIX_EPOCH_TIME_FORMAT.match(v)
-          Sequel.string_to_datetime(v)
+          Sequel.database_to_application_timestamp(v)
         end
         db.translator.add_translator("timestamp", &prok)
         db.translator.add_translator("datetime", &prok)

data/lib/sequel/core.rb

@@ -29,15 +29,58 @@
 #
 #   Sequel.datetime_class = DateTime
 #
+# Sequel doesn't pay much attention to timezones by default, but you can set it
+# handle timezones if you want.  There are three separate timezone settings:
+#
+# * application_timezone - The timezone you want the application to use.  This is
+#   the timezone that incoming times from the database and typecasting are converted
+#   to.
+# * database_timezone - The timezone for storage in the database.  This is the
+#   timezone to which Sequel will convert timestamps before literalizing them
+#   for storage in the database.  It is also the timezone that Sequel will assume
+#   database timestamp values are already in (if they don't include an offset).
+# * typecast_timezone - The timezone that incoming data that Sequel needs to typecast
+#   is assumed to be already in (if they don't include an offset).
+#
+# You can set also set all three timezones to the same value at once via
+# Sequel.default_timezone=.
+#
+#   Sequel.application_timezone = :utc # or :local or nil
+#   Sequel.database_timezone = :utc # or :local or nil
+#   Sequel.typecast_timezone = :utc # or :local or nil
+#   Sequel.default_timezone = :utc # or :local or nil
+#
+# The only timezone values that are supported by default are :utc (convert to UTC),
+# :local (convert to local time), and nil (don't convert).  If you need to
+# convert to a specific timezone, or need the timezones being used to change based
+# on the environment (e.g. current user), you need to use an extension (and use
+# DateTime as the datetime_class).
+#
 # You can set the SEQUEL_NO_CORE_EXTENSIONS constant or environment variable to have
 # Sequel not extend the core classes.
 module Sequel
+  # The offset of the current time zone from UTC, in seconds.
+  LOCAL_DATETIME_OFFSET_SECS = Time.now.utc_offset
+  
+  # The offset of the current time zone from UTC, as a fraction of a day.
+  LOCAL_DATETIME_OFFSET = respond_to?(:Rational, true) ? Rational(LOCAL_DATETIME_OFFSET_SECS, 60*60*24) : LOCAL_DATETIME_OFFSET_SECS/60/60/24.0
+  
+  @application_timezone = nil
   @convert_two_digit_years = true
+  @database_timezone = nil
   @datetime_class = Time
+  @typecast_timezone = nil
   @virtual_row_instance_eval = true
   
   class << self
     attr_accessor :convert_two_digit_years, :datetime_class, :virtual_row_instance_eval
+    attr_accessor :application_timezone, :database_timezone, :typecast_timezone
+  end
+  
+  # Convert the given Time/DateTime object into the database timezone, used when
+  # literalizing objects in an SQL string.
+  def self.application_to_database_timestamp(v)
+    convert_output_timestamp(v, Sequel.database_timezone)
   end
 
   # Returns true if the passed object could be a specifier of conditions, false otherwise.
@@ -73,6 +116,29 @@ module Sequel
     Database.connect(*args, &block)
   end
   
+  # Convert the given object into an object of Sequel.datetime_class in the
+  # application_timezone.  Used when coverting datetime/timestamp columns
+  # returned by the database.
+  def self.database_to_application_timestamp(v)
+    convert_timestamp(v, Sequel.database_timezone)
+  end
+  
+  # Sets the database, application, and typecasting timezones to the given timezone. 
+  def self.default_timezone=(tz)
+    self.database_timezone = tz
+    self.application_timezone = tz
+    self.typecast_timezone = tz
+  end
+  
+  # Load all Sequel extensions given.  Only loads extensions included in this
+  # release of Sequel, doesn't load external extensions.
+  #
+  #   Sequel.extension(:schema_dumper)
+  #   Sequel.extension(:pagination, :query)
+  def self.extension(*extensions)
+    require(extensions, 'extensions')
+  end
+  
   # Set the method to call on identifiers going into the database.  This affects
   # the literalization of identifiers by calling this method on them before they are input.
   # Sequel upcases identifiers in all SQL strings for most databases, so to turn that off:
@@ -112,15 +178,6 @@ module Sequel
     Database.quote_identifiers = value
   end
 
-  # Load all Sequel extensions given.  Only loads extensions included in this
-  # release of Sequel, doesn't load external extensions.
-  #
-  #   Sequel.extension(:schema_dumper)
-  #   Sequel.extension(:pagination, :query)
-  def self.extension(*extensions)
-    require(extensions, 'extensions')
-  end
-
   # Require all given files which should be in the same or a subdirectory of
   # this file.  If a subdir is given, assume all files are in that subdir.
   def self.require(files, subdir=nil)
@@ -168,7 +225,14 @@ module Sequel
       raise InvalidValue, "Invalid Time value #{s.inspect} (#{e.message})"
     end
   end
-
+  
+  # Convert the given object into an object of Sequel.datetime_class in the
+  # application_timezone.  Used when typecasting values when assigning them
+  # to model datetime attributes.
+  def self.typecast_to_application_timestamp(v)
+    convert_timestamp(v, Sequel.typecast_timezone)
+  end
+  
   ### Private Class Methods ###
 
   # Helper method that the database adapter class methods that are added to Sequel via
@@ -184,6 +248,78 @@ module Sequel
     end
     connect(opts, &block)
   end
+  
+  # Converts the object from a String, Array, Date, DateTime, or Time into an
+  # instance of Sequel.datetime_class.  If a string and an offset is not given,
+  # assume that the string is already in the given input_timezone.
+  def self.convert_input_timestamp(v, input_timezone) # :nodoc:
+    case v
+    when String
+      v2 = Sequel.string_to_datetime(v)
+      if !input_timezone || Date._parse(v).has_key?(:offset)
+        v2
+      else
+        # Correct for potentially wrong offset if offset is given
+        if v2.is_a?(DateTime)
+          # DateTime assumes UTC if no offset is given
+          v2 = v2.new_offset(LOCAL_DATETIME_OFFSET) - LOCAL_DATETIME_OFFSET if input_timezone == :local
+        else
+          # Time assumes local time if no offset is given
+          v2 = v2.getutc + LOCAL_DATETIME_OFFSET_SECS if input_timezone == :utc
+        end
+        v2
+      end
+    when Array
+      y, mo, d, h, mi, s = v
+      if datetime_class == DateTime
+        DateTime.civil(y, mo, d, h, mi, s, input_timezone == :utc ? 0 : LOCAL_DATETIME_OFFSET)
+      else
+        Time.send(input_timezone == :utc ? :utc : :local, y, mo, d, h, mi, s)
+      end
+    when Time
+      if datetime_class == DateTime
+        v.respond_to?(:to_datetime) ? v.to_datetime : string_to_datetime(v.iso8601)
+      else
+        v
+      end
+    when DateTime
+      if datetime_class == DateTime
+        v
+      else
+        v.respond_to?(:to_time) ? v.to_time : string_to_datetime(v.to_s)
+      end
+    when Date
+      convert_input_timestamp(v.to_s, input_timezone)
+    else
+      raise InvalidValue, "Invalid convert_input_timestamp type: #{v.inspect}"
+    end
+  end
+  
+  # Converts the object to the given output_timezone.
+  def self.convert_output_timestamp(v, output_timezone) # :nodoc:
+    if output_timezone
+      if v.is_a?(DateTime)
+        v.new_offset(output_timezone == :utc ? 0 : LOCAL_DATETIME_OFFSET)
+      else
+        v.send(output_timezone == :utc ? :getutc : :getlocal)
+      end
+    else
+      v
+    end
+  end
+  
+  # Converts the given object from the given input timezone to the
+  # application timezone using convert_input_timestamp and
+  # convert_output_timestamp.
+  def self.convert_timestamp(v, input_timezone) # :nodoc:
+    begin
+      convert_output_timestamp(convert_input_timestamp(v, input_timezone), Sequel.application_timezone)
+    rescue InvalidValue
+      raise
+    rescue
+      raise InvalidValue, "Invalid #{datetime_class} value: #{v.inspect}"
+    end
+  end
 
   # Method that adds a database adapter class method to Sequel that calls
   # Sequel.adapter_method.
@@ -193,7 +329,7 @@ module Sequel
     end
   end
 
-  private_class_method :adapter_method, :def_adapter_method
+  private_class_method :adapter_method, :convert_input_timestamp, :convert_output_timestamp, :convert_timestamp, :def_adapter_method
   
   require(%w"metaprogramming sql connection_pool exceptions dataset database version")
   require(%w"schema_generator schema_methods schema_sql", 'database')

data/lib/sequel/database.rb

@@ -234,9 +234,10 @@ module Sequel
     
     ### Instance Methods ###
 
-    # Executes the supplied SQL statement string.
+    # Runs the supplied SQL statement string on the database server.
+    # Alias for run.
     def <<(sql)
-      execute_ddl(sql)
+      run(sql)
     end
     
     # Returns a dataset from the database. If the first argument is a string,
@@ -429,6 +430,14 @@ module Sequel
       @quote_identifiers = @opts.include?(:quote_identifiers) ? @opts[:quote_identifiers] : (@@quote_identifiers.nil? ? quote_identifiers_default : @@quote_identifiers)
     end
     
+    # Runs the supplied SQL statement string on the database server. Returns nil.
+    # Options:
+    # * :server - The server to run the SQL on.
+    def run(sql, opts={})
+      execute_ddl(sql, opts)
+      nil
+    end
+    
     # Returns a new dataset with the select method invoked.
     def select(*args, &block)
       dataset.select(*args, &block)
@@ -924,6 +933,8 @@ module Sequel
         Date.new(value.year, value.month, value.day)
       when String
         Sequel.string_to_date(value)
+      when Hash
+        Date.new(*[:year, :month, :day].map{|x| (value[x] || value[x.to_s]).to_i})
       else
         raise InvalidValue, "invalid value for Date: #{value.inspect}"
       end
@@ -931,14 +942,12 @@ module Sequel
 
     # Typecast the value to a DateTime or Time depending on Sequel.datetime_class
     def typecast_value_datetime(value)
-      raise(Sequel::InvalidValue, "invalid value for Datetime: #{value.inspect}") unless [DateTime, Date, Time, String].any?{|c| value.is_a?(c)}
-      if Sequel.datetime_class === value
-        # Already the correct class, no need to convert
-       value
+      raise(Sequel::InvalidValue, "invalid value for Datetime: #{value.inspect}") unless [DateTime, Date, Time, String, Hash].any?{|c| value.is_a?(c)}
+      klass = Sequel.datetime_class
+      if value.is_a?(Hash)
+        klass.send(klass == Time ? :mktime : :new, *[:year, :month, :day, :hour, :minute, :second].map{|x| (value[x] || value[x.to_s]).to_i})
       else
-        # First convert it to standard ISO 8601 time, then
-        # parse that string using the time class.
-        Sequel.string_to_datetime(Time === value ? value.iso8601 : value.to_s)
+        Sequel.typecast_to_application_timestamp(value)
       end
     end
 
@@ -976,6 +985,9 @@ module Sequel
         value
       when String
         Sequel.string_to_time(value)
+      when Hash
+        t = Time.now
+        Time.mktime(t.year, t.month, t.day, *[:hour, :minute, :second].map{|x| (value[x] || value[x.to_s]).to_i})
       else
         raise Sequel::InvalidValue, "invalid value for Time: #{value.inspect}"
       end

data/lib/sequel/dataset.rb

@@ -42,13 +42,13 @@ module Sequel
 
     # All methods that should have a ! method added that modifies
     # the receiver.
-    MUTATION_METHODS = %w'add_graph_aliases and distinct exclude exists
+    MUTATION_METHODS = %w'add_graph_aliases and distinct except exclude
     filter from from_self full_outer_join graph
-    group group_and_count group_by having inner_join intersect invert join
-    left_outer_join limit naked or order order_by order_more paginate qualify query reject
-    reverse reverse_order right_outer_join select select_all select_more
-    set_defaults set_graph_aliases set_overrides sort sort_by
-    unfiltered ungraphed union unordered where with with_sql'.collect{|x| x.to_sym}
+    group group_and_count group_by having inner_join intersect invert join join_table
+    left_outer_join limit naked or order order_by order_more paginate qualify query
+    reverse reverse_order right_outer_join select select_all select_more server
+    set_defaults set_graph_aliases set_overrides unfiltered ungraphed ungrouped union
+    unlimited unordered where with with_recursive with_sql'.collect{|x| x.to_sym}
 
     NOTIMPL_MSG = "This method must be overridden in Sequel adapters".freeze
     WITH_SUPPORTED='with'.freeze
@@ -175,6 +175,10 @@ module Sequel
     
     # Iterates over the records in the dataset as they are yielded from the
     # database adapter, and returns self.
+    #
+    # Note that this method is not safe to use on many adapters if you are
+    # running additional queries inside the provided block.  If you are
+    # running queries inside the block, you use should all instead of each.
     def each(&block)
       if @opts[:graph]
         graph_each(&block)
@@ -275,10 +279,25 @@ module Sequel
       true
     end
     
+    # Whether the dataset supports timezones in literal timestamps
+    def supports_timestamp_timezones?
+      false
+    end
+    
+    # Whether the dataset supports fractional seconds in literal timestamps
+    def supports_timestamp_usecs?
+      true
+    end
+    
     # Whether the dataset supports window functions.
     def supports_window_functions?
       false
     end
+    
+    # Truncates the dataset.  Returns nil.
+    def truncate
+      execute_ddl(truncate_sql)
+    end
 
     # Updates values for the dataset.  The returned value is generally the
     # number of rows updated, but that is adapter dependent.
@@ -314,6 +333,12 @@ module Sequel
       @db.execute(sql, {:server=>@opts[:server] || :read_only}.merge(opts), &block)
     end
     
+    # Execute the given SQL on the database using execute_ddl.
+    def execute_ddl(sql, opts={}, &block)
+      @db.execute_ddl(sql, default_server_opts(opts), &block)
+      nil
+    end
+    
     # Execute the given SQL on the database using execute_dui.
     def execute_dui(sql, opts={}, &block)
       @db.execute_dui(sql, default_server_opts(opts), &block)

data/lib/sequel/dataset/convenience.rb

@@ -109,7 +109,7 @@ module Sequel
     #   # this will commit every 50 records
     #   dataset.import([:x, :y], [[1, 2], [3, 4], ...], :slice => 50)
     def import(columns, values, opts={})
-      return @db.transaction{execute_dui("#{insert_sql_base}#{quote_schema_table(@opts[:from].first)} (#{identifier_list(columns)}) VALUES #{literal(values)}")} if values.is_a?(Dataset)
+      return @db.transaction{execute_dui("#{insert_sql_base}#{quote_schema_table(@opts[:from].first)} (#{identifier_list(columns)}) #{subselect_sql(values)}")} if values.is_a?(Dataset)
 
       return if values.empty?
       raise(Error, IMPORT_ERROR_MSG) if columns.empty?

data/lib/sequel/dataset/sql.rb

@@ -17,6 +17,8 @@ module Sequel
     QUESTION_MARK = '?'.freeze
     STOCK_COUNT_OPTS = {:select => [SQL::AliasedExpression.new(LiteralString.new("COUNT(*)").freeze, :count)], :order => nil}.freeze
     SELECT_CLAUSE_ORDER = %w'with distinct columns from join where group having compounds order limit'.freeze
+    TIMESTAMP_FORMAT = "'%Y-%m-%d %H:%M:%S%N%z'".freeze
+    STANDARD_TIMESTAMP_FORMAT = "TIMESTAMP #{TIMESTAMP_FORMAT}".freeze
     TWO_ARITY_OPERATORS = ::Sequel::SQL::ComplexExpression::TWO_ARITY_OPERATORS
     WILDCARD = '*'.freeze
     SQL_WITH = "WITH ".freeze
@@ -88,6 +90,11 @@ module Sequel
         raise(InvalidOperation, "invalid operator #{op}")
       end
     end
+    
+    # SQL fragment for constants
+    def constant_sql(constant)
+      constant.to_s
+    end
 
     # Returns the number of records in the dataset.
     def count
@@ -103,11 +110,7 @@ module Sequel
 
       return static_sql(opts[:sql]) if opts[:sql]
 
-      if opts[:group]
-        raise InvalidOperation, "Grouped datasets cannot be deleted from"
-      elsif opts[:from].is_a?(Array) && opts[:from].size > 1
-        raise InvalidOperation, "Joined datasets cannot be deleted from"
-      end
+      check_modification_allowed!
 
       sql = "DELETE FROM #{source_list(opts[:from])}"
 
@@ -308,7 +311,7 @@ module Sequel
     #   dataset.group(:id) # SELECT * FROM items GROUP BY id
     #   dataset.group(:id, :name) # SELECT * FROM items GROUP BY id, name
     def group(*columns)
-      clone(:group => columns)
+      clone(:group => (columns.compact.empty? ? nil : columns))
     end
     alias group_by group
 
@@ -344,6 +347,8 @@ module Sequel
     def insert_sql(*values)
       return static_sql(@opts[:sql]) if @opts[:sql]
 
+      check_modification_allowed!
+
       from = source_list(@opts[:from])
       case values.size
       when 0
@@ -778,14 +783,32 @@ module Sequel
     def subscript_sql(s)
       "#{literal(s.f)}[#{expression_list(s.sub)}]"
     end
+    
+    # SQL query to truncate the table
+    def truncate_sql
+      if opts[:sql]
+        static_sql(opts[:sql])
+      else
+        check_modification_allowed!
+        raise(InvalidOperation, "Can't truncate filtered datasets") if opts[:where]
+        _truncate_sql(source_list(opts[:from]))
+      end
+    end
 
     # Returns a copy of the dataset with no filters (HAVING or WHERE clause) applied.
     # 
-    #   dataset.group(:a).having(:a=>1).where(:b).unfiltered # SELECT * FROM items
+    #   dataset.group(:a).having(:a=>1).where(:b).unfiltered # SELECT * FROM items GROUP BY a
     def unfiltered
       clone(:where => nil, :having => nil)
     end
 
+    # Returns a copy of the dataset with no grouping (GROUP or HAVING clause) applied.
+    # 
+    #   dataset.group(:a).having(:a=>1).where(:b).ungrouped # SELECT * FROM items WHERE b
+    def ungrouped
+      clone(:group => nil, :having => nil)
+    end
+
     # Adds a UNION clause using a second dataset object.
     # A UNION compound dataset returns all rows in either the current dataset
     # or the given dataset.
@@ -826,11 +849,7 @@ module Sequel
 
       return static_sql(opts[:sql]) if opts[:sql]
 
-      if opts[:group]
-        raise InvalidOperation, "A grouped dataset cannot be updated"
-      elsif (opts[:from].size > 1) or opts[:join]
-        raise InvalidOperation, "A joined dataset cannot be updated"
-      end
+      check_modification_allowed!
       
       sql = "UPDATE #{source_list(@opts[:from])} SET "
       set = if values.is_a?(Hash)
@@ -945,6 +964,13 @@ module Sequel
     def as_sql(expression, aliaz)
       "#{expression} AS #{quote_identifier(aliaz)}"
     end
+    
+    # Raise an InvalidOperation exception if deletion is not allowed
+    # for this dataset
+    def check_modification_allowed!
+      raise(InvalidOperation, "Grouped datasets cannot be modified") if opts[:group]
+      raise(InvalidOperation, "Joined datasets cannot be modified") if (opts[:from].is_a?(Array) && opts[:from].size > 1) || opts[:join]
+    end
 
     # Converts an array of column names into a comma seperated string of 
     # column names. If the array is empty, a wildcard (*) is returned.
@@ -969,6 +995,11 @@ module Sequel
       columns.map{|i| literal(i)}.join(COMMA_SEPARATOR)
     end
     
+    # The strftime format to use when literalizing the time.
+    def default_timestamp_format
+      requires_sql_standard_datetimes? ? STANDARD_TIMESTAMP_FORMAT : TIMESTAMP_FORMAT
+    end
+    
     # SQL fragment based on the expr type.  See #filter.
     def filter_expr(expr = nil, &block)
       expr = nil if expr == []
@@ -1002,6 +1033,27 @@ module Sequel
         raise(Error, 'Invalid filter argument')
       end
     end
+    
+    # Format the timestamp based on the default_timestamp_format, with a couple
+    # of modifiers.  First, allow %N to be used for fractions seconds (if the
+    # database supports them), and override %z to always use a numeric offset
+    # of hours and minutes.
+    def format_timestamp(v)
+      v2 = Sequel.application_to_database_timestamp(v)
+      fmt = default_timestamp_format.gsub(/%[Nz]/) do |m|
+        if m == '%N'
+          sprintf(".%06d", v.is_a?(DateTime) ? v.sec_fraction*86400000000 : v.usec) if supports_timestamp_usecs?
+        else
+          if supports_timestamp_timezones?
+            # Would like to just use %z format, but it doesn't appear to work on Windows
+            # Instead, the offset fragment is constructed manually
+            minutes = (v2.is_a?(DateTime) ? v2.offset * 1440 : v2.utc_offset/60).to_i
+            sprintf("%+03i%02i", *minutes.divmod(60))
+          end
+        end
+      end
+      v2.strftime(fmt)
+    end
 
     # SQL fragment specifying a list of identifiers
     def identifier_list(columns)
@@ -1035,7 +1087,7 @@ module Sequel
       order.map do |f|
         case f
         when SQL::OrderedExpression
-          SQL::OrderedExpression.new(f.expression, !f.descending)
+          f.invert
         else
           SQL::OrderedExpression.new(f)
         end
@@ -1074,9 +1126,9 @@ module Sequel
       requires_sql_standard_datetimes? ? v.strftime("DATE '%Y-%m-%d'") : "'#{v}'"
     end
 
-    # SQL fragment for DateTime, using the ISO8601 format.
+    # SQL fragment for DateTime
     def literal_datetime(v)
-      requires_sql_standard_datetimes? ? v.strftime("TIMESTAMP '%Y-%m-%d %H:%M:%S'") : "'#{v}'"
+      format_timestamp(v)
     end
 
     # SQL fragment for SQL::Expression, result depends on the specific type of expression.
@@ -1128,12 +1180,12 @@ module Sequel
       c_alias ? as_sql(qc, c_alias) : qc
     end
 
-    # SQL fragment for Time, uses the ISO8601 format.
+    # SQL fragment for Time
     def literal_time(v)
-      requires_sql_standard_datetimes? ? v.strftime("TIMESTAMP '%Y-%m-%d %H:%M:%S'") : "'#{v.iso8601}'"
+      format_timestamp(v)
     end
 
-    # SQL fragment for true.
+    # SQL fragment for true
     def literal_true
       BOOL_TRUE
     end
@@ -1323,5 +1375,11 @@ module Sequel
     def table_ref(t)
       t.is_a?(String) ? quote_identifier(t) : literal(t)
     end
+    
+    # Formats the truncate statement.  Assumes the table given has already been
+    # literalized.
+    def _truncate_sql(table)
+      "TRUNCATE TABLE #{table}"
+    end
   end
 end