groupdate 4.1.2 → 5.2.0

data/lib/groupdate/enumerable.rb

@@ -1,31 +1,25 @@
 module Enumerable
   Groupdate::PERIODS.each do |period|
-    define_method :"group_by_#{period}" do |*args, &block|
+    define_method :"group_by_#{period}" do |*args, **options, &block|
       if block
-        Groupdate::Magic::Enumerable.group_by(self, period, args[0] || {}, &block)
+        raise ArgumentError, "wrong number of arguments (given #{args.size}, expected 0)" if args.any?
+        Groupdate::Magic::Enumerable.group_by(self, period, options, &block)
       elsif respond_to?(:scoping)
-        scoping { @klass.send(:"group_by_#{period}", *args, &block) }
+        scoping { @klass.group_by_period(period, *args, **options, &block) }
       else
         raise ArgumentError, "no block given"
       end
     end
   end
 
-  def group_by_period(*args, &block)
+  def group_by_period(period, *args, **options, &block)
     if block || !respond_to?(:scoping)
-      period = args[0]
-      options = args[1] || {}
+      raise ArgumentError, "wrong number of arguments (given #{args.size + 1}, expected 1)" if args.any?
 
-      options = options.dup
-      # to_sym is unsafe on user input, so convert to strings
-      permitted_periods = ((options.delete(:permit) || Groupdate::PERIODS).map(&:to_sym) & Groupdate::PERIODS).map(&:to_s)
-      if permitted_periods.include?(period.to_s)
-        send("group_by_#{period}", options, &block)
-      else
-        raise ArgumentError, "Unpermitted period"
-      end
+      Groupdate::Magic.validate_period(period, options.delete(:permit))
+      send("group_by_#{period}", **options, &block)
     else
-      scoping { @klass.send(:group_by_period, *args, &block) }
+      scoping { @klass.group_by_period(period, *args, **options, &block) }
     end
   end
 end

data/lib/groupdate/magic.rb

@@ -2,18 +2,52 @@ require "i18n"
 
 module Groupdate
   class Magic
-    attr_accessor :period, :options, :group_index
+    DAYS = [:monday, :tuesday, :wednesday, :thursday, :friday, :saturday, :sunday]
+
+    attr_accessor :period, :options, :group_index, :n_seconds
 
     def initialize(period:, **options)
       @period = period
       @options = options
 
-      unknown_keywords = options.keys - [:day_start, :time_zone, :dates, :series, :week_start, :format, :locale, :range, :reverse]
+      validate_keywords
+      validate_arguments
+
+      if options[:n]
+        raise ArgumentError, "n must be a positive integer" if !options[:n].is_a?(Integer) || options[:n] < 1
+        @period = :custom
+        @n_seconds = options[:n].to_i
+        @n_seconds *= 60 if period == :minute
+      end
+    end
+
+    def validate_keywords
+      known_keywords = [:time_zone, :dates, :series, :format, :locale, :range, :reverse]
+
+      if %i[week day_of_week].include?(period)
+        known_keywords << :week_start
+      end
+
+      if %i[day week month quarter year day_of_week hour_of_day day_of_month day_of_year month_of_year].include?(period)
+        known_keywords << :day_start
+      else
+        # prevent Groupdate.day_start from applying
+        @day_start = 0
+      end
+
+      if %i[second minute].include?(period)
+        known_keywords << :n
+      end
+
+      unknown_keywords = options.keys - known_keywords
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
+    end
 
-      raise Groupdate::Error, "Unrecognized time zone" unless time_zone
-      raise Groupdate::Error, "Unrecognized :week_start option" if period == :week && !week_start
-      raise Groupdate::Error, "Cannot use endless range for :range option" if options[:range].is_a?(Range) && !options[:range].end
+    def validate_arguments
+      # TODO better messages
+      raise ArgumentError, "Unrecognized time zone" unless time_zone
+      raise ArgumentError, "Unrecognized :week_start option" unless week_start
+      raise ArgumentError, ":day_start must be between 0 and 24" if (day_start / 3600) < 0 || (day_start / 3600) >= 24
     end
 
     def time_zone
@@ -25,7 +59,10 @@ module Groupdate
     end
 
     def week_start
-      @week_start ||= [:mon, :tue, :wed, :thu, :fri, :sat, :sun].index((options[:week_start] || options[:start] || Groupdate.week_start).to_sym)
+      @week_start ||= begin
+        v = (options[:week_start] || Groupdate.week_start).to_sym
+        DAYS.index(v) || [:mon, :tue, :wed, :thu, :fri, :sat, :sun].index(v)
+      end
     end
 
     def day_start
@@ -39,7 +76,8 @@ module Groupdate
           period: period,
           time_zone: time_zone,
           day_start: day_start,
-          week_start: week_start
+          week_start: week_start,
+          n_seconds: n_seconds
         )
     end
 
@@ -47,6 +85,11 @@ module Groupdate
       series_builder.time_range
     end
 
+    def self.validate_period(period, permit)
+      permitted_periods = ((permit || Groupdate::PERIODS).map(&:to_sym) & Groupdate::PERIODS).map(&:to_s)
+      raise ArgumentError, "Unpermitted period" unless permitted_periods.include?(period.to_s)
+    end
+
     class Enumerable < Magic
       def group_by(enum, &_block)
         group = enum.group_by do |v|
@@ -85,10 +128,10 @@ module Groupdate
       def cast_method
         @cast_method ||= begin
           case period
+          when :minute_of_hour, :hour_of_day, :day_of_month, :day_of_year, :month_of_year
+            lambda { |k| k.to_i }
           when :day_of_week
             lambda { |k| (k.to_i - 1 - week_start) % 7 }
-          when :hour_of_day, :day_of_month, :month_of_year, :minute_of_hour
-            lambda { |k| k.to_i }
           else
             utc = ActiveSupport::TimeZone["UTC"]
             lambda { |k| (k.is_a?(String) || !k.respond_to?(:to_time) ? utc.parse(k.to_s) : k.to_time).in_time_zone(time_zone) }
@@ -142,7 +185,8 @@ module Groupdate
             time_zone: magic.time_zone,
             time_range: magic.time_range,
             week_start: magic.week_start,
-            day_start: magic.day_start
+            day_start: magic.day_start,
+            n_seconds: magic.n_seconds
           ).generate
 
         # add Groupdate info

data/lib/groupdate/query_methods.rb

@@ -1,25 +1,18 @@
 module Groupdate
   module QueryMethods
     Groupdate::PERIODS.each do |period|
-      define_method :"group_by_#{period}" do |field, time_zone = nil, range = nil, **options|
+      define_method :"group_by_#{period}" do |field, **options|
         Groupdate::Magic::Relation.generate_relation(self,
           period: period,
           field: field,
-          time_zone: time_zone,
-          range: range,
           **options
         )
       end
     end
 
     def group_by_period(period, field, permit: nil, **options)
-      # to_sym is unsafe on user input, so convert to strings
-      permitted_periods = ((permit || Groupdate::PERIODS).map(&:to_sym) & Groupdate::PERIODS).map(&:to_s)
-      if permitted_periods.include?(period.to_s)
-        send("group_by_#{period}", field, **options)
-      else
-        raise ArgumentError, "Unpermitted period"
-      end
+      Groupdate::Magic.validate_period(period, permit)
+      send("group_by_#{period}", field, **options)
     end
   end
 end

data/lib/groupdate/relation_builder.rb

@@ -1,8 +1,11 @@
 module Groupdate
   class RelationBuilder
-    attr_reader :period, :column, :day_start, :week_start
+    attr_reader :period, :column, :day_start, :week_start, :n_seconds
+
+    def initialize(relation, column:, period:, time_zone:, time_range:, week_start:, day_start:, n_seconds:)
+      # very important
+      validate_column(column)
 
-    def initialize(relation, column:, period:, time_zone:, time_range:, week_start:, day_start:)
       @relation = relation
       @column = resolve_column(relation, column)
       @period = period
@@ -10,6 +13,7 @@ module Groupdate
       @time_range = time_range
       @week_start = week_start
       @day_start = day_start
+      @n_seconds = n_seconds
 
       if relation.default_timezone == :local
         raise Groupdate::Error, "ActiveRecord::Base.default_timezone must be :utc to use Groupdate"
@@ -27,22 +31,28 @@ module Groupdate
       adapter_name = @relation.connection.adapter_name
       query =
         case adapter_name
-        when "MySQL", "Mysql2", "Mysql2Spatial", 'Mysql2Rgeo'
+        when "Mysql2", "Mysql2Spatial", "Mysql2Rgeo"
+          day_start_column = "CONVERT_TZ(#{column}, '+00:00', ?) - INTERVAL ? second"
+
           case period
-          when :day_of_week
-            ["DAYOFWEEK(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?)) - 1", time_zone]
-          when :hour_of_day
-            ["(EXTRACT(HOUR from CONVERT_TZ(#{column}, '+00:00', ?)) + 24 - #{day_start / 3600}) % 24", time_zone]
           when :minute_of_hour
-            ["(EXTRACT(MINUTE from CONVERT_TZ(#{column}, '+00:00', ?)))", time_zone]
+            ["MINUTE(#{day_start_column})", time_zone, day_start]
+          when :hour_of_day
+            ["HOUR(#{day_start_column})", time_zone, day_start]
+          when :day_of_week
+            ["DAYOFWEEK(#{day_start_column}) - 1", time_zone, day_start]
           when :day_of_month
-            ["DAYOFMONTH(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?))", time_zone]
+            ["DAYOFMONTH(#{day_start_column})", time_zone, day_start]
+          when :day_of_year
+            ["DAYOFYEAR(#{day_start_column})", time_zone, day_start]
           when :month_of_year
-            ["MONTH(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?))", time_zone]
+            ["MONTH(#{day_start_column})", time_zone, day_start]
           when :week
-            ["CONVERT_TZ(DATE_FORMAT(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL ((#{7 - week_start} + WEEKDAY(CONVERT_TZ(#{column}, '+00:00', ?) - INTERVAL #{day_start} second)) % 7) DAY) - INTERVAL #{day_start} second, '+00:00', ?), '%Y-%m-%d 00:00:00') + INTERVAL #{day_start} second, ?, '+00:00')", time_zone, time_zone, time_zone]
+            ["CONVERT_TZ(DATE_FORMAT(#{day_start_column} - INTERVAL ((? + DAYOFWEEK(#{day_start_column})) % 7) DAY, '%Y-%m-%d 00:00:00') + INTERVAL ? second, ?, '+00:00')", time_zone, day_start, 12 - week_start, time_zone, day_start, day_start, time_zone]
           when :quarter
-            ["DATE_ADD(CONVERT_TZ(DATE_FORMAT(DATE(CONCAT(EXTRACT(YEAR FROM CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?)), '-', LPAD(1 + 3 * (QUARTER(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?)) - 1), 2, '00'), '-01')), '%Y-%m-%d %H:%i:%S'), ?, '+00:00'), INTERVAL #{day_start} second)", time_zone, time_zone, time_zone]
+            ["CONVERT_TZ(DATE_FORMAT(DATE(CONCAT(YEAR(#{day_start_column}), '-', LPAD(1 + 3 * (QUARTER(#{day_start_column}) - 1), 2, '00'), '-01')), '%Y-%m-%d %H:%i:%S') + INTERVAL ? second, ?, '+00:00')", time_zone, day_start, time_zone, day_start, day_start, time_zone]
+          when :custom
+            ["FROM_UNIXTIME((UNIX_TIMESTAMP(#{column}) DIV ?) * ?)", n_seconds, n_seconds]
           else
             format =
               case period
@@ -60,43 +70,58 @@ module Groupdate
                 "%Y-01-01 00:00:00"
               end
 
-            ["DATE_ADD(CONVERT_TZ(DATE_FORMAT(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?), '#{format}'), ?, '+00:00'), INTERVAL #{day_start} second)", time_zone, time_zone]
+            ["CONVERT_TZ(DATE_FORMAT(#{day_start_column}, ?) + INTERVAL ? second, ?, '+00:00')", time_zone, day_start, format, day_start, time_zone]
           end
         when "PostgreSQL", "PostGIS"
+          day_start_column = "#{column}::timestamptz AT TIME ZONE ? - INTERVAL ?"
+          day_start_interval = "#{day_start} second"
+
           case period
-          when :day_of_week
-            ["EXTRACT(DOW from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
-          when :hour_of_day
-            ["EXTRACT(HOUR from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
           when :minute_of_hour
-            ["EXTRACT(MINUTE from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
+            ["EXTRACT(MINUTE FROM #{day_start_column})::integer", time_zone, day_start_interval]
+          when :hour_of_day
+            ["EXTRACT(HOUR FROM #{day_start_column})::integer", time_zone, day_start_interval]
+          when :day_of_week
+            ["EXTRACT(DOW FROM #{day_start_column})::integer", time_zone, day_start_interval]
           when :day_of_month
-            ["EXTRACT(DAY from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
+            ["EXTRACT(DAY FROM #{day_start_column})::integer", time_zone, day_start_interval]
+          when :day_of_year
+            ["EXTRACT(DOY FROM #{day_start_column})::integer", time_zone, day_start_interval]
           when :month_of_year
-            ["EXTRACT(MONTH from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
-          when :week # start on Sunday, not PostgreSQL default Monday
-            ["(DATE_TRUNC('#{period}', (#{column}::timestamptz - INTERVAL '#{week_start} day' - INTERVAL '#{day_start} second') AT TIME ZONE ?) + INTERVAL '#{week_start} day' + INTERVAL '#{day_start} second') AT TIME ZONE ?", time_zone, time_zone]
+            ["EXTRACT(MONTH FROM #{day_start_column})::integer", time_zone, day_start_interval]
+          when :week
+            ["(DATE_TRUNC('day', #{day_start_column} - INTERVAL '1 day' * ((? + EXTRACT(DOW FROM #{day_start_column})::integer) % 7)) + INTERVAL ?) AT TIME ZONE ?", time_zone, day_start_interval, 13 - week_start, time_zone, day_start_interval, day_start_interval, time_zone]
+          when :custom
+            ["TO_TIMESTAMP(FLOOR(EXTRACT(EPOCH FROM #{column}::timestamptz) / ?) * ?)", n_seconds, n_seconds]
           else
-            ["(DATE_TRUNC('#{period}', (#{column}::timestamptz - INTERVAL '#{day_start} second') AT TIME ZONE ?) + INTERVAL '#{day_start} second') AT TIME ZONE ?", time_zone, time_zone]
+            if day_start == 0
+              # prettier
+              ["DATE_TRUNC(?, #{day_start_column}) AT TIME ZONE ?", period, time_zone, day_start_interval, time_zone]
+            else
+              ["(DATE_TRUNC(?, #{day_start_column}) + INTERVAL ?) AT TIME ZONE ?", period, time_zone, day_start_interval, day_start_interval, time_zone]
+            end
           end
         when "SQLite"
           raise Groupdate::Error, "Time zones not supported for SQLite" unless @time_zone.utc_offset.zero?
           raise Groupdate::Error, "day_start not supported for SQLite" unless day_start.zero?
-          raise Groupdate::Error, "week_start not supported for SQLite" unless week_start == 6
 
           if period == :week
-            ["strftime('%%Y-%%m-%%d 00:00:00 UTC', #{column}, '-6 days', 'weekday 0')"]
+            ["strftime('%Y-%m-%d 00:00:00 UTC', #{column}, '-6 days', ?)", "weekday #{(week_start + 1) % 7}"]
+          elsif period == :custom
+            ["datetime((strftime('%s', #{column}) / ?) * ?, 'unixepoch')", n_seconds, n_seconds]
           else
             format =
               case period
-              when :hour_of_day
-                "%H"
               when :minute_of_hour
                 "%M"
+              when :hour_of_day
+                "%H"
               when :day_of_week
                 "%w"
               when :day_of_month
                 "%d"
+              when :day_of_year
+                "%j"
               when :month_of_year
                 "%m"
               when :second
@@ -115,37 +140,40 @@ module Groupdate
                 "%Y-01-01 00:00:00 UTC"
               end
 
-            ["strftime('#{format.gsub(/%/, '%%')}', #{column})"]
+            ["strftime(?, #{column})", format]
           end
         when "Redshift"
+          day_start_column = "CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL ?"
+          day_start_interval = "#{day_start} second"
+
           case period
-          when :day_of_week
-            ["EXTRACT(DOW from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
-          when :hour_of_day
-            ["EXTRACT(HOUR from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
           when :minute_of_hour
-            ["EXTRACT(MINUTE from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
+            ["EXTRACT(MINUTE from #{day_start_column})::integer", time_zone, day_start_interval]
+          when :hour_of_day
+            ["EXTRACT(HOUR from #{day_start_column})::integer", time_zone, day_start_interval]
+          when :day_of_week
+            ["EXTRACT(DOW from #{day_start_column})::integer", time_zone, day_start_interval]
           when :day_of_month
-            ["EXTRACT(DAY from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
+            ["EXTRACT(DAY from #{day_start_column})::integer", time_zone, day_start_interval]
+          when :day_of_year
+            ["EXTRACT(DOY from #{day_start_column})::integer", time_zone, day_start_interval]
           when :month_of_year
-            ["EXTRACT(MONTH from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
+            ["EXTRACT(MONTH from #{day_start_column})::integer", time_zone, day_start_interval]
           when :week # start on Sunday, not Redshift default Monday
             # Redshift does not return timezone information; it
             # always says it is in UTC time, so we must convert
             # back to UTC to play properly with the rest of Groupdate.
-            #
-            ["CONVERT_TIMEZONE(?, 'Etc/UTC', DATE_TRUNC(?, CONVERT_TIMEZONE(?, #{column}) - INTERVAL '#{week_start} day' - INTERVAL '#{day_start} second'))::timestamp + INTERVAL '#{week_start} day' + INTERVAL '#{day_start} second'", time_zone, period, time_zone]
+            week_start_interval = "#{week_start} day"
+            ["CONVERT_TIMEZONE(?, 'Etc/UTC', DATE_TRUNC('week', #{day_start_column} - INTERVAL ?) + INTERVAL ? + INTERVAL ?)::timestamp", time_zone, time_zone, day_start_interval, week_start_interval, week_start_interval, day_start_interval]
+          when :custom
+            raise Groupdate::Error, "Not implemented yet"
           else
-            ["CONVERT_TIMEZONE(?, 'Etc/UTC', DATE_TRUNC(?, CONVERT_TIMEZONE(?, #{column}) - INTERVAL '#{day_start} second'))::timestamp + INTERVAL '#{day_start} second'", time_zone, period, time_zone]
+            ["CONVERT_TIMEZONE(?, 'Etc/UTC', DATE_TRUNC(?, #{day_start_column}) + INTERVAL ?)::timestamp", time_zone, period, time_zone, day_start_interval, day_start_interval]
           end
         else
           raise Groupdate::Error, "Connection adapter not supported: #{adapter_name}"
         end
 
-      if adapter_name == "MySQL" && period == :week
-        query[0] = "CAST(#{query[0]} AS DATETIME)"
-      end
-
       clause = @relation.send(:sanitize_sql_array, query)
 
       # cleaner queries in logs
@@ -158,22 +186,35 @@ module Groupdate
     end
 
     def clean_group_clause_mysql(clause)
-      clause = clause.gsub("DATE_SUB(#{column}, INTERVAL 0 second)", "#{column}")
-      if clause.start_with?("DATE_ADD(") && clause.end_with?(", INTERVAL 0 second)")
-        clause = clause[9..-21]
-      end
-      clause
+      clause.gsub(/ (\-|\+) INTERVAL 0 second/, "")
     end
 
     def where_clause
       if @time_range.is_a?(Range)
-        op = @time_range.exclude_end? ? "<" : "<="
-        ["#{column} >= ? AND #{column} #{op} ?", @time_range.first, @time_range.last]
+        if @time_range.end
+          op = @time_range.exclude_end? ? "<" : "<="
+          if @time_range.begin
+            ["#{column} >= ? AND #{column} #{op} ?", @time_range.first, @time_range.last]
+          else
+            ["#{column} #{op} ?", @time_range.last]
+          end
+        else
+          ["#{column} >= ?", @time_range.first]
+        end
       else
         ["#{column} IS NOT NULL"]
       end
     end
 
+    # basic version of Active Record disallow_raw_sql!
+    # symbol = column (safe), Arel node = SQL (safe), other = untrusted
+    def validate_column(column)
+      # matches table.column and column
+      unless column.is_a?(Symbol) || column.is_a?(Arel::Nodes::SqlLiteral) || /\A\w+(\.\w+)?\z/i.match(column.to_s)
+        warn "[groupdate] Non-attribute argument: #{column}. Use Arel.sql() for known-safe values. This will raise an error in Groupdate 6"
+      end
+    end
+
     # resolves eagerly
     # need to convert both where_clause (easy)
     # and group_clause (not easy) if want to avoid this

data/lib/groupdate/series_builder.rb

@@ -1,29 +1,64 @@
 module Groupdate
   class SeriesBuilder
-    attr_reader :period, :time_zone, :day_start, :week_start, :options
+    attr_reader :period, :time_zone, :day_start, :week_start, :n_seconds, :options
 
     CHECK_PERIODS = [:day, :week, :month, :quarter, :year]
 
-    def initialize(period:, time_zone:, day_start:, week_start:, **options)
+    def initialize(period:, time_zone:, day_start:, week_start:, n_seconds:, **options)
       @period = period
       @time_zone = time_zone
       @week_start = week_start
       @day_start = day_start
+      @n_seconds = n_seconds
       @options = options
-      @round_time = {}
+      @week_start_key = Groupdate::Magic::DAYS[@week_start] if @week_start
     end
 
     def generate(data, default_value:, series_default: true, multiple_groups: false, group_index: nil)
       series = generate_series(data, multiple_groups, group_index)
       series = handle_multiple(data, series, multiple_groups, group_index)
 
+      verified_data = {}
+      series.each do |k|
+        verified_data[k] = data.delete(k)
+      end
+
+      # this is a fun one
+      # PostgreSQL and Ruby both return the 2nd hour when converting/parsing a backward DST change
+      # Other databases and Active Support return the 1st hour (as expected)
+      # Active Support good: ActiveSupport::TimeZone["America/Los_Angeles"].parse("2013-11-03 01:00:00")
+      # MySQL good: SELECT CONVERT_TZ('2013-11-03 01:00:00', 'America/Los_Angeles', 'Etc/UTC');
+      # Ruby not good: Time.parse("2013-11-03 01:00:00")
+      # PostgreSQL not good: SELECT '2013-11-03 01:00:00'::timestamp AT TIME ZONE 'America/Los_Angeles';
+      # we need to account for this here
+      if series_default && CHECK_PERIODS.include?(period)
+        data.each do |k, v|
+          key = multiple_groups ? k[group_index] : k
+          # TODO only do this for PostgreSQL
+          # this may mask some inconsistent time zone errors
+          # but not sure there's a better approach
+          if key.hour == (key - 1.hour).hour && series.include?(key - 1.hour)
+            key -= 1.hour
+            if multiple_groups
+              k[group_index]  = key
+            else
+              k = key
+            end
+            verified_data[k] = v
+          elsif key != round_time(key)
+            # only need to show what database returned since it will cast in Ruby time zone
+            raise Groupdate::Error, "Database and Ruby have inconsistent time zone info. Database returned #{key}"
+          end
+        end
+      end
+
       unless entire_series?(series_default)
-        series = series.select { |k| data[k] }
+        series = series.select { |k| verified_data[k] }
       end
 
       value = 0
       result = Hash[series.map do |k|
-        value = data.delete(k) || (@options[:carry_forward] && value) || default_value
+        value = verified_data[k] || (@options[:carry_forward] && value) || default_value
         key =
           if multiple_groups
             k[0...group_index] + [key_format.call(k[group_index])] + k[(group_index + 1)..-1]
@@ -34,20 +69,21 @@ module Groupdate
         [key, value]
       end]
 
-      # only check for database
-      # only checks remaining keys to avoid expensive calls to round_time
-      if series_default && CHECK_PERIODS.include?(period)
-        check_consistent_time_zone_info(data, multiple_groups, group_index)
-      end
-
       result
     end
 
     def round_time(time)
+      if period == :custom
+        return time_zone.at((time.to_time.to_i / n_seconds) * n_seconds)
+      end
+
       time = time.to_time.in_time_zone(time_zone)
 
-      # only if day_start != 0 for performance
-      time -= day_start.seconds if day_start != 0
+      if day_start != 0
+        # apply day_start to a time object that's not affected by DST
+        time = change_zone.call(time, utc)
+        time -= day_start.seconds
+      end
 
       time =
         case period
@@ -60,9 +96,7 @@ module Groupdate
         when :day
           time.beginning_of_day
         when :week
-          # same logic as MySQL group
-          weekday = (time.wday - 1) % 7
-          (time - ((7 - week_start + weekday) % 7).days).midnight
+          time.beginning_of_week(@week_start_key)
         when :month
           time.beginning_of_month
         when :quarter
@@ -74,33 +108,54 @@ module Groupdate
         when :minute_of_hour
           time.min
         when :day_of_week
-          (time.wday - 1 - week_start) % 7
+          time.days_to_week_start(@week_start_key)
         when :day_of_month
           time.day
         when :month_of_year
           time.month
+        when :day_of_year
+          time.yday
         else
           raise Groupdate::Error, "Invalid period"
         end
 
-      # only if day_start != 0 for performance
-      time += day_start.seconds if day_start != 0 && time.is_a?(Time)
+      if day_start != 0 && time.is_a?(Time)
+        time += day_start.seconds
+        time = change_zone.call(time, time_zone)
+      end
 
       time
     end
 
+    def change_zone
+      @change_zone ||= begin
+        if ActiveSupport::VERSION::STRING >= "5.2"
+          ->(time, zone) { time.change(zone: zone) }
+        else
+          # TODO make more efficient
+          ->(time, zone) { zone.parse(time.strftime("%Y-%m-%d %H:%M:%S")) }
+        end
+      end
+    end
+
     def time_range
       @time_range ||= begin
         time_range = options[:range]
-        if time_range.is_a?(Range) && time_range.first.is_a?(Date)
-          # convert range of dates to range of times
-          # use parsing instead of in_time_zone due to Rails < 4
-          last = time_zone.parse(time_range.last.to_s)
-          last += 1.day unless time_range.exclude_end?
-          time_range = Range.new(time_zone.parse(time_range.first.to_s), last, true)
+        # entire range must be Date if begin or end is Date
+        if time_range.is_a?(Range) && (time_range.begin.is_a?(Date) || time_range.end.is_a?(Date))
+          if time_range.begin
+            start = time_zone.parse(time_range.first.to_s)
+          end
+          if time_range.end
+            last = time_zone.parse(time_range.last.to_s)
+            last += 1.day unless time_range.exclude_end?
+          end
+          time_range = Range.new(start, last, true)
         elsif !time_range && options[:last]
           if period == :quarter
             step = 3.months
+          elsif period == :custom
+            step = n_seconds
           elsif 1.respond_to?(period)
             step = 1.send(period)
           else
@@ -117,7 +172,8 @@ module Groupdate
               if options[:current] == false
                 round_time(start_at - step)...round_time(now)
               else
-                round_time(start_at)..now
+                # extend to end of current period
+                round_time(start_at)...(round_time(now) + step)
               end
           end
         end
@@ -141,12 +197,14 @@ module Groupdate
         0..59
       when :day_of_month
         1..31
+      when :day_of_year
+        1..366
       when :month_of_year
         1..12
       else
         time_range = self.time_range
         time_range =
-          if time_range.is_a?(Range)
+          if time_range.is_a?(Range) && time_range.begin && time_range.end
             time_range
           else
             # use first and last values
@@ -157,26 +215,41 @@ module Groupdate
                 data.keys.sort
               end
 
-            tr = sorted_keys.first..sorted_keys.last
-            if options[:current] == false && sorted_keys.any? && round_time(now) >= tr.last
-              tr = tr.first...round_time(now)
+            if time_range.is_a?(Range)
+              if sorted_keys.any?
+                if time_range.begin
+                  time_range.begin..sorted_keys.last
+                else
+                  Range.new(sorted_keys.first, time_range.end, time_range.exclude_end?)
+                end
+              else
+                nil..nil
+              end
+            else
+              tr = sorted_keys.first..sorted_keys.last
+              if options[:current] == false && sorted_keys.any? && round_time(now) >= tr.last
+                tr = tr.first...round_time(now)
+              end
+              tr
             end
-            tr
           end
 
-        if time_range.first
-          series = [round_time(time_range.first)]
+        if time_range.begin
+          series = [round_time(time_range.begin)]
 
           if period == :quarter
             step = 3.months
+          elsif period == :custom
+            step = n_seconds
           else
             step = 1.send(period)
           end
 
           last_step = series.last
+          day_start_hour = day_start / 3600
           loop do
             next_step = last_step + step
-            next_step = round_time(next_step) if next_step.hour != day_start # add condition to speed up
+            next_step = round_time(next_step) if next_step.hour != day_start_hour # add condition to speed up
             break unless time_range.cover?(next_step)
 
             if next_step == last_step
@@ -195,34 +268,36 @@ module Groupdate
     end
 
     def key_format
-      locale = options[:locale] || I18n.locale
-      use_dates = options.key?(:dates) ? options[:dates] : Groupdate.dates
+      @key_format ||= begin
+        locale = options[:locale] || I18n.locale
+        use_dates = options.key?(:dates) ? options[:dates] : Groupdate.dates
 
-      if options[:format]
-        if options[:format].respond_to?(:call)
-          options[:format]
-        else
-          sunday = time_zone.parse("2014-03-02 00:00:00")
-          lambda do |key|
-            case period
-            when :hour_of_day
-              key = sunday + key.hours + day_start.seconds
-            when :minute_of_hour
-              key = sunday + key.minutes + day_start.seconds
-            when :day_of_week
-              key = sunday + key.days + (week_start + 1).days
-            when :day_of_month
-              key = Date.new(2014, 1, key).to_time
-            when :month_of_year
-              key = Date.new(2014, key, 1).to_time
+        if options[:format]
+          if options[:format].respond_to?(:call)
+            options[:format]
+          else
+            sunday = time_zone.parse("2014-03-02 00:00:00")
+            lambda do |key|
+              case period
+              when :hour_of_day
+                key = sunday + key.hours + day_start.seconds
+              when :minute_of_hour
+                key = sunday + key.minutes + day_start.seconds
+              when :day_of_week
+                key = sunday + key.days + (week_start + 1).days
+              when :day_of_month
+                key = Date.new(2014, 1, key).to_time
+              when :month_of_year
+                key = Date.new(2014, key, 1).to_time
+              end
+              I18n.localize(key, format: options[:format], locale: locale)
             end
-            I18n.localize(key, format: options[:format], locale: locale)
           end
+        elsif [:day, :week, :month, :quarter, :year].include?(period) && use_dates
+          lambda { |k| k.to_date }
+        else
+          lambda { |k| k }
         end
-      elsif [:day, :week, :month, :quarter, :year].include?(period) && use_dates
-        lambda { |k| k.to_date }
-      else
-        lambda { |k| k }
       end
     end
 
@@ -242,23 +317,12 @@ module Groupdate
       end
     end
 
-    def check_consistent_time_zone_info(data, multiple_groups, group_index)
-      keys = data.keys
-      if multiple_groups
-        keys.map! { |k| k[group_index] }
-        keys.uniq!
-      end
-
-      keys.each do |key|
-        if key != round_time(key)
-          # only need to show what database returned since it will cast in Ruby time zone
-          raise Groupdate::Error, "Database and Ruby have inconsistent time zone info. Database returned #{key}"
-        end
-      end
-    end
-
     def entire_series?(series_default)
       options.key?(:series) ? options[:series] : series_default
     end
+
+    def utc
+      @utc ||= ActiveSupport::TimeZone["Etc/UTC"]
+    end
   end
 end