groupdate2 4.1.5 → 5.0.0

data/lib/groupdate/query_methods.rb

@@ -1,25 +1,18 @@
 module Groupdate
   module QueryMethods
     Groupdate::PERIODS.each do |period|
-      define_method :"group_by_#{period}" do |field, time_zone = nil, range = nil, **options|
+      define_method :"group_by_#{period}" do |field, **options|
         Groupdate::Magic::Relation.generate_relation(self,
           period: period,
           field: field,
-          time_zone: time_zone,
-          range: range,
           **options
         )
       end
     end
 
     def group_by_period(period, field, permit: nil, **options)
-      # to_sym is unsafe on user input, so convert to strings
-      permitted_periods = ((permit || Groupdate::PERIODS).map(&:to_sym) & Groupdate::PERIODS).map(&:to_s)
-      if permitted_periods.include?(period.to_s)
-        send("group_by_#{period}", field, **options)
-      else
-        raise ArgumentError, "Unpermitted period"
-      end
+      Groupdate::Magic.validate_period(period, permit)
+      send("group_by_#{period}", field, **options)
     end
   end
 end

data/lib/groupdate/relation.rb

@@ -13,4 +13,11 @@ module Groupdate
       Groupdate.process_result(self, super, default_value: default_value)
     end
   end
+
+  module RelationRecords
+    def records
+      super
+      Groupdate.process_series_label(self, @records)
+    end
+  end
 end

data/lib/groupdate/relation_builder.rb

@@ -3,9 +3,9 @@ require_relative 'sql_server_group_clause'
 module Groupdate
   class RelationBuilder
     include SqlServerGroupClause
-    attr_reader :period, :column, :day_start, :week_start
+    attr_reader :period, :column, :day_start, :week_start, :series_label
 
-    def initialize(relation, column:, period:, time_zone:, time_range:, week_start:, day_start:)
+    def initialize(relation, column:, period:, time_zone:, time_range:, week_start:, day_start:, series_label:)
       @relation = relation
       @column = resolve_column(relation, column)
       @period = period
@@ -13,6 +13,7 @@ module Groupdate
       @time_range = time_range
       @week_start = week_start
       @day_start = day_start
+      @series_label = series_label
 
       if relation.default_timezone == :local
         raise Groupdate::Error, "ActiveRecord::Base.default_timezone must be :utc to use Groupdate"
@@ -20,7 +21,10 @@ module Groupdate
     end
 
     def generate
-      @relation.group(group_clause).where(*where_clause)
+      group_by = group_clause
+      @relation = @relation.group(group_by).where(*where_clause)
+      @relation.select_values += ["#{group_by} AS #{series_label}"] if series_label
+      @relation
     end
 
     private
@@ -32,22 +36,26 @@ module Groupdate
         case adapter_name
         when "SQLServer"
           sql_server_group_clause(time_zone)
-        when "MySQL", "Mysql2", "Mysql2Spatial", 'Mysql2Rgeo'
+        when "Mysql2", "Mysql2Spatial", "Mysql2Rgeo"
+          day_start_column = "CONVERT_TZ(#{column}, '+00:00', ?) - INTERVAL ? second"
+
           case period
-          when :day_of_week
-            ["DAYOFWEEK(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?)) - 1", time_zone]
-          when :hour_of_day
-            ["(EXTRACT(HOUR from CONVERT_TZ(#{column}, '+00:00', ?)) + 24 - #{day_start / 3600}) % 24", time_zone]
           when :minute_of_hour
-            ["(EXTRACT(MINUTE from CONVERT_TZ(#{column}, '+00:00', ?)))", time_zone]
+            ["MINUTE(#{day_start_column})", time_zone, day_start]
+          when :hour_of_day
+            ["HOUR(#{day_start_column})", time_zone, day_start]
+          when :day_of_week
+            ["DAYOFWEEK(#{day_start_column}) - 1", time_zone, day_start]
           when :day_of_month
-            ["DAYOFMONTH(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?))", time_zone]
+            ["DAYOFMONTH(#{day_start_column})", time_zone, day_start]
+          when :day_of_year
+            ["DAYOFYEAR(#{day_start_column})", time_zone, day_start]
           when :month_of_year
-            ["MONTH(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?))", time_zone]
+            ["MONTH(#{day_start_column})", time_zone, day_start]
           when :week
-            ["CONVERT_TZ(DATE_FORMAT(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL ((#{7 - week_start} + WEEKDAY(CONVERT_TZ(#{column}, '+00:00', ?) - INTERVAL #{day_start} second)) % 7) DAY) - INTERVAL #{day_start} second, '+00:00', ?), '%Y-%m-%d 00:00:00') + INTERVAL #{day_start} second, ?, '+00:00')", time_zone, time_zone, time_zone]
+            ["CONVERT_TZ(DATE_FORMAT(#{day_start_column} - INTERVAL ((? + DAYOFWEEK(#{day_start_column})) % 7) DAY, '%Y-%m-%d 00:00:00') + INTERVAL ? second, ?, '+00:00')", time_zone, day_start, 12 - week_start, time_zone, day_start, day_start, time_zone]
           when :quarter
-            ["DATE_ADD(CONVERT_TZ(DATE_FORMAT(DATE(CONCAT(EXTRACT(YEAR FROM CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?)), '-', LPAD(1 + 3 * (QUARTER(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?)) - 1), 2, '00'), '-01')), '%Y-%m-%d %H:%i:%S'), ?, '+00:00'), INTERVAL #{day_start} second)", time_zone, time_zone, time_zone]
+            ["CONVERT_TZ(DATE_FORMAT(DATE(CONCAT(YEAR(#{day_start_column}), '-', LPAD(1 + 3 * (QUARTER(#{day_start_column}) - 1), 2, '00'), '-01')), '%Y-%m-%d %H:%i:%S') + INTERVAL ? second, ?, '+00:00')", time_zone, day_start, time_zone, day_start, day_start, time_zone]
           else
             format =
               case period
@@ -65,43 +73,54 @@ module Groupdate
                 "%Y-01-01 00:00:00"
               end
 
-            ["DATE_ADD(CONVERT_TZ(DATE_FORMAT(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?), '#{format}'), ?, '+00:00'), INTERVAL #{day_start} second)", time_zone, time_zone]
+            ["CONVERT_TZ(DATE_FORMAT(#{day_start_column}, ?) + INTERVAL ? second, ?, '+00:00')", time_zone, day_start, format, day_start, time_zone]
           end
         when "PostgreSQL", "PostGIS"
+          day_start_column = "#{column}::timestamptz AT TIME ZONE ? - INTERVAL ?"
+          day_start_interval = "#{day_start} second"
+
           case period
-          when :day_of_week
-            ["EXTRACT(DOW from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
-          when :hour_of_day
-            ["EXTRACT(HOUR from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
           when :minute_of_hour
-            ["EXTRACT(MINUTE from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
+            ["EXTRACT(MINUTE FROM #{day_start_column})::integer", time_zone, day_start_interval]
+          when :hour_of_day
+            ["EXTRACT(HOUR FROM #{day_start_column})::integer", time_zone, day_start_interval]
+          when :day_of_week
+            ["EXTRACT(DOW FROM #{day_start_column})::integer", time_zone, day_start_interval]
           when :day_of_month
-            ["EXTRACT(DAY from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
+            ["EXTRACT(DAY FROM #{day_start_column})::integer", time_zone, day_start_interval]
+          when :day_of_year
+            ["EXTRACT(DOY FROM #{day_start_column})::integer", time_zone, day_start_interval]
           when :month_of_year
-            ["EXTRACT(MONTH from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
-          when :week # start on Sunday, not PostgreSQL default Monday
-            ["(DATE_TRUNC('#{period}', (#{column}::timestamptz - INTERVAL '#{week_start} day' - INTERVAL '#{day_start} second') AT TIME ZONE ?) + INTERVAL '#{week_start} day' + INTERVAL '#{day_start} second') AT TIME ZONE ?", time_zone, time_zone]
+            ["EXTRACT(MONTH FROM #{day_start_column})::integer", time_zone, day_start_interval]
+          when :week
+            ["(DATE_TRUNC('day', #{day_start_column} - INTERVAL '1 day' * ((? + EXTRACT(DOW FROM #{day_start_column})::integer) % 7)) + INTERVAL ?) AT TIME ZONE ?", time_zone, day_start_interval, 13 - week_start, time_zone, day_start_interval, day_start_interval, time_zone]
           else
-            ["(DATE_TRUNC('#{period}', (#{column}::timestamptz - INTERVAL '#{day_start} second') AT TIME ZONE ?) + INTERVAL '#{day_start} second') AT TIME ZONE ?", time_zone, time_zone]
+            if day_start == 0
+              # prettier
+              ["DATE_TRUNC(?, #{day_start_column}) AT TIME ZONE ?", period, time_zone, day_start_interval, time_zone]
+            else
+              ["(DATE_TRUNC(?, #{day_start_column}) + INTERVAL ?) AT TIME ZONE ?", period, time_zone, day_start_interval, day_start_interval, time_zone]
+            end
           end
         when "SQLite"
           raise Groupdate::Error, "Time zones not supported for SQLite" unless @time_zone.utc_offset.zero?
           raise Groupdate::Error, "day_start not supported for SQLite" unless day_start.zero?
-          raise Groupdate::Error, "week_start not supported for SQLite" unless week_start == 6
 
           if period == :week
-            ["strftime('%%Y-%%m-%%d 00:00:00 UTC', #{column}, '-6 days', 'weekday 0')"]
+            ["strftime('%Y-%m-%d 00:00:00 UTC', #{column}, '-6 days', ?)", "weekday #{(week_start + 1) % 7}"]
           else
             format =
               case period
-              when :hour_of_day
-                "%H"
               when :minute_of_hour
                 "%M"
+              when :hour_of_day
+                "%H"
               when :day_of_week
                 "%w"
               when :day_of_month
                 "%d"
+              when :day_of_year
+                "%j"
               when :month_of_year
                 "%m"
               when :second
@@ -120,37 +139,38 @@ module Groupdate
                 "%Y-01-01 00:00:00 UTC"
               end
 
-            ["strftime('#{format.gsub(/%/, '%%')}', #{column})"]
+            ["strftime(?, #{column})", format]
           end
         when "Redshift"
+          day_start_column = "CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL ?"
+          day_start_interval = "#{day_start} second"
+
           case period
-          when :day_of_week
-            ["EXTRACT(DOW from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
-          when :hour_of_day
-            ["EXTRACT(HOUR from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
           when :minute_of_hour
-            ["EXTRACT(MINUTE from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
+            ["EXTRACT(MINUTE from #{day_start_column})::integer", time_zone, day_start_interval]
+          when :hour_of_day
+            ["EXTRACT(HOUR from #{day_start_column})::integer", time_zone, day_start_interval]
+          when :day_of_week
+            ["EXTRACT(DOW from #{day_start_column})::integer", time_zone, day_start_interval]
           when :day_of_month
-            ["EXTRACT(DAY from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
+            ["EXTRACT(DAY from #{day_start_column})::integer", time_zone, day_start_interval]
+          when :day_of_year
+            ["EXTRACT(DOY from #{day_start_column})::integer", time_zone, day_start_interval]
           when :month_of_year
-            ["EXTRACT(MONTH from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
+            ["EXTRACT(MONTH from #{day_start_column})::integer", time_zone, day_start_interval]
           when :week # start on Sunday, not Redshift default Monday
             # Redshift does not return timezone information; it
             # always says it is in UTC time, so we must convert
             # back to UTC to play properly with the rest of Groupdate.
-            #
-            ["CONVERT_TIMEZONE(?, 'Etc/UTC', DATE_TRUNC(?, CONVERT_TIMEZONE(?, #{column}) - INTERVAL '#{week_start} day' - INTERVAL '#{day_start} second'))::timestamp + INTERVAL '#{week_start} day' + INTERVAL '#{day_start} second'", time_zone, period, time_zone]
+            week_start_interval = "#{week_start} day"
+            ["CONVERT_TIMEZONE(?, 'Etc/UTC', DATE_TRUNC('week', #{day_start_column} - INTERVAL ?) + INTERVAL ? + INTERVAL ?)::timestamp", time_zone, time_zone, day_start_interval, week_start_interval, week_start_interval, day_start_interval]
           else
-            ["CONVERT_TIMEZONE(?, 'Etc/UTC', DATE_TRUNC(?, CONVERT_TIMEZONE(?, #{column}) - INTERVAL '#{day_start} second'))::timestamp + INTERVAL '#{day_start} second'", time_zone, period, time_zone]
+            ["CONVERT_TIMEZONE(?, 'Etc/UTC', DATE_TRUNC(?, #{day_start_column}) + INTERVAL ?)::timestamp", time_zone, period, time_zone, day_start_interval, day_start_interval]
           end
         else
           raise Groupdate::Error, "Connection adapter not supported: #{adapter_name}"
         end
 
-      if adapter_name == "MySQL" && period == :week
-        query[0] = "CAST(#{query[0]} AS DATETIME)"
-      end
-
       clause = @relation.send(:sanitize_sql_array, query)
 
       # cleaner queries in logs
@@ -163,11 +183,7 @@ module Groupdate
     end
 
     def clean_group_clause_mysql(clause)
-      clause = clause.gsub("DATE_SUB(#{column}, INTERVAL 0 second)", "#{column}")
-      if clause.start_with?("DATE_ADD(") && clause.end_with?(", INTERVAL 0 second)")
-        clause = clause[9..-21]
-      end
-      clause
+      clause.gsub(/ (\-|\+) INTERVAL 0 second/, "")
     end
 
     def where_clause

data/lib/groupdate/series_builder.rb

@@ -11,19 +11,55 @@ module Groupdate
       @day_start = day_start
       @options = options
       @round_time = {}
+      @week_start_key = Groupdate::Magic::DAYS[@week_start] if @week_start
     end
 
     def generate(data, default_value:, series_default: true, multiple_groups: false, group_index: nil)
       series = generate_series(data, multiple_groups, group_index)
       series = handle_multiple(data, series, multiple_groups, group_index)
 
+      verified_data = {}
+      series.each do |k|
+        verified_data[k] = data.delete(k)
+      end
+
+      # this is a fun one
+      # PostgreSQL and Ruby both return the 2nd hour when converting/parsing a backward DST change
+      # Other databases and Active Support return the 1st hour (as expected)
+      # Active Support good: ActiveSupport::TimeZone["America/Los_Angeles"].parse("2013-11-03 01:00:00")
+      # MySQL good: SELECT CONVERT_TZ('2013-11-03 01:00:00', 'America/Los_Angeles', 'Etc/UTC');
+      # SQLServer good: SELECT CAST('2013-11-03 01:00:00' AS DATETIME2(3)) AT TIME ZONE 'Pacific Standard Time' AT TIME ZONE 'UTC'
+      # Ruby not good: Time.parse("2013-11-03 01:00:00")
+      # PostgreSQL not good: SELECT '2013-11-03 01:00:00'::timestamp AT TIME ZONE 'America/Los_Angeles';
+      # we need to account for this here
+      if series_default && CHECK_PERIODS.include?(period)
+        data.each do |k, v|
+          key = multiple_groups ? k[group_index] : k
+          # TODO only do this for PostgreSQL
+          # this may mask some inconsistent time zone errors
+          # but not sure there's a better approach
+          if key.hour == (key - 1.hour).hour && series.include?(key - 1.hour)
+            key -= 1.hour
+            if multiple_groups
+              k[group_index]  = key
+            else
+              k = key
+            end
+            verified_data[k] = v
+          elsif key != round_time(key)
+            # only need to show what database returned since it will cast in Ruby time zone
+            # raise Groupdate::Error, "Database and Ruby have inconsistent time zone info. Database returned #{key}: #{round_time(key)}"
+          end
+        end
+      end
+
       unless entire_series?(series_default)
-        series = series.select { |k| data[k] }
+        series = series.select { |k| verified_data[k] }
       end
 
       value = 0
       result = Hash[series.map do |k|
-        value = data.delete(k) || (@options[:carry_forward] && value) || default_value
+        value = verified_data[k] || (@options[:carry_forward] && value) || default_value
         key =
           if multiple_groups
             k[0...group_index] + [key_format.call(k[group_index])] + k[(group_index + 1)..-1]
@@ -34,20 +70,22 @@ module Groupdate
         [key, value]
       end]
 
-      # only check for database
-      # only checks remaining keys to avoid expensive calls to round_time
-      if series_default && CHECK_PERIODS.include?(period)
-        check_consistent_time_zone_info(data, multiple_groups, group_index)
-      end
-
       result
     end
 
+    def format_series_label(series_label)
+      series_label = round_time(series_label) if series_label.respond_to?(:to_time)
+      key_format.call(series_label)
+    end
+
     def round_time(time)
       time = time.to_time.in_time_zone(time_zone)
 
-      # only if day_start != 0 for performance
-      time -= day_start.seconds if day_start != 0
+      if day_start != 0
+        # apply day_start to a time object that's not affected by DST
+        time = change_zone.call(time, utc)
+        time -= day_start.seconds
+      end
 
       time =
         case period
@@ -60,9 +98,7 @@ module Groupdate
         when :day
           time.beginning_of_day
         when :week
-          # same logic as MySQL group
-          weekday = (time.wday - 1) % 7
-          (time - ((7 - week_start + weekday) % 7).days).midnight
+          time.beginning_of_week(@week_start_key)
         when :month
           time.beginning_of_month
         when :quarter
@@ -74,30 +110,44 @@ module Groupdate
         when :minute_of_hour
           time.min
         when :day_of_week
-          (time.wday - 1 - week_start) % 7
+          time.days_to_week_start(@week_start_key)
         when :day_of_month
           time.day
         when :month_of_year
           time.month
+        when :day_of_year
+          time.yday
         else
           raise Groupdate::Error, "Invalid period"
         end
 
-      # only if day_start != 0 for performance
-      time += day_start.seconds if day_start != 0 && time.is_a?(Time)
+      if day_start != 0 && time.is_a?(Time)
+        time += day_start.seconds
+        time = change_zone.call(time, time_zone)
+      end
 
       time
     end
 
+    def change_zone
+      @change_zone ||= begin
+        if ActiveSupport::VERSION::STRING >= "5.2"
+          ->(time, zone) { time.change(zone: zone) }
+        else
+          # TODO make more efficient
+          ->(time, zone) { zone.parse(time.strftime("%Y-%m-%d %H:%M:%S")) }
+        end
+      end
+    end
+
     def time_range
       @time_range ||= begin
         time_range = options[:range]
         if time_range.is_a?(Range) && time_range.first.is_a?(Date)
           # convert range of dates to range of times
-          # use parsing instead of in_time_zone due to Rails < 4
-          last = time_zone.parse(time_range.last.to_s)
+          last = time_range.last.in_time_zone(time_zone)
           last += 1.day unless time_range.exclude_end?
-          time_range = Range.new(time_zone.parse(time_range.first.to_s), last, true)
+          time_range = Range.new(time_range.first.in_time_zone(time_zone), last, true)
         elsif !time_range && options[:last]
           if period == :quarter
             step = 3.months
@@ -117,7 +167,8 @@ module Groupdate
               if options[:current] == false
                 round_time(start_at - step)...round_time(now)
               else
-                round_time(start_at)..now
+                # extend to end of current period
+                round_time(start_at)...(round_time(now) + step)
               end
           end
         end
@@ -141,6 +192,8 @@ module Groupdate
         0..59
       when :day_of_month
         1..31
+      when :day_of_year
+        1..366
       when :month_of_year
         1..12
       else
@@ -164,8 +217,8 @@ module Groupdate
             tr
           end
 
-        if time_range.first
-          series = [round_time(time_range.first)]
+        if time_range.begin
+          series = [round_time(time_range.begin)]
 
           if period == :quarter
             step = 3.months
@@ -174,9 +227,10 @@ module Groupdate
           end
 
           last_step = series.last
+          day_start_hour = day_start / 3600
           loop do
             next_step = last_step + step
-            next_step = round_time(next_step) if next_step.hour != day_start # add condition to speed up
+            next_step = round_time(next_step) if next_step.hour != day_start_hour # add condition to speed up
             break unless time_range.cover?(next_step)
 
             if next_step == last_step
@@ -195,34 +249,36 @@ module Groupdate
     end
 
     def key_format
-      locale = options[:locale] || I18n.locale
-      use_dates = options.key?(:dates) ? options[:dates] : Groupdate.dates
+      @key_format ||= begin
+        locale = options[:locale] || I18n.locale
+        use_dates = options.key?(:dates) ? options[:dates] : Groupdate.dates
 
-      if options[:format]
-        if options[:format].respond_to?(:call)
-          options[:format]
-        else
-          sunday = time_zone.parse("2014-03-02 00:00:00")
-          lambda do |key|
-            case period
-            when :hour_of_day
-              key = sunday + key.hours + day_start.seconds
-            when :minute_of_hour
-              key = sunday + key.minutes + day_start.seconds
-            when :day_of_week
-              key = sunday + key.days + (week_start + 1).days
-            when :day_of_month
-              key = Date.new(2014, 1, key).to_time
-            when :month_of_year
-              key = Date.new(2014, key, 1).to_time
+        if options[:format]
+          if options[:format].respond_to?(:call)
+            options[:format]
+          else
+            sunday = time_zone.parse("2014-03-02 00:00:00")
+            lambda do |key|
+              case period
+              when :hour_of_day
+                key = sunday + key.hours + day_start.seconds
+              when :minute_of_hour
+                key = sunday + key.minutes + day_start.seconds
+              when :day_of_week
+                key = sunday + key.days + (week_start + 1).days
+              when :day_of_month
+                key = Date.new(2014, 1, key).to_time
+              when :month_of_year
+                key = Date.new(2014, key, 1).to_time
+              end
+              I18n.localize(key, format: options[:format], locale: locale)
             end
-            I18n.localize(key, format: options[:format], locale: locale)
           end
+        elsif [:day, :week, :month, :quarter, :year].include?(period) && use_dates
+          lambda { |k| k.to_date }
+        else
+          lambda { |k| k }
         end
-      elsif [:day, :week, :month, :quarter, :year].include?(period) && use_dates
-        lambda { |k| k.to_date }
-      else
-        lambda { |k| k }
       end
     end
 
@@ -242,23 +298,12 @@ module Groupdate
       end
     end
 
-    def check_consistent_time_zone_info(data, multiple_groups, group_index)
-      keys = data.keys
-      if multiple_groups
-        keys.map! { |k| k[group_index] }
-        keys.uniq!
-      end
-
-      keys.each do |key|
-        if key != round_time(key)
-          # only need to show what database returned since it will cast in Ruby time zone
-          raise Groupdate::Error, "Database and Ruby have inconsistent time zone info. Database returned #{key} != #{round_time(key)}"
-        end
-      end
-    end
-
     def entire_series?(series_default)
       options.key?(:series) ? options[:series] : series_default
     end
+
+    def utc
+      @utc ||= ActiveSupport::TimeZone["Etc/UTC"]
+    end
   end
 end