groupdate2 4.1.5

data/lib/groupdate/query_methods.rb

@@ -0,0 +1,25 @@
+module Groupdate
+  module QueryMethods
+    Groupdate::PERIODS.each do |period|
+      define_method :"group_by_#{period}" do |field, time_zone = nil, range = nil, **options|
+        Groupdate::Magic::Relation.generate_relation(self,
+          period: period,
+          field: field,
+          time_zone: time_zone,
+          range: range,
+          **options
+        )
+      end
+    end
+
+    def group_by_period(period, field, permit: nil, **options)
+      # to_sym is unsafe on user input, so convert to strings
+      permitted_periods = ((permit || Groupdate::PERIODS).map(&:to_sym) & Groupdate::PERIODS).map(&:to_s)
+      if permitted_periods.include?(period.to_s)
+        send("group_by_#{period}", field, **options)
+      else
+        raise ArgumentError, "Unpermitted period"
+      end
+    end
+  end
+end

data/lib/groupdate/relation.rb

@@ -0,0 +1,16 @@
+require "active_support/concern"
+
+module Groupdate
+  module Relation
+    extend ActiveSupport::Concern
+
+    included do
+      attr_accessor :groupdate_values
+    end
+
+    def calculate(*args, &block)
+      default_value = [:count, :sum].include?(args[0]) ? 0 : nil
+      Groupdate.process_result(self, super, default_value: default_value)
+    end
+  end
+end

data/lib/groupdate/relation_builder.rb

@@ -0,0 +1,191 @@
+require_relative 'sql_server_group_clause'
+
+module Groupdate
+  class RelationBuilder
+    include SqlServerGroupClause
+    attr_reader :period, :column, :day_start, :week_start
+
+    def initialize(relation, column:, period:, time_zone:, time_range:, week_start:, day_start:)
+      @relation = relation
+      @column = resolve_column(relation, column)
+      @period = period
+      @time_zone = time_zone
+      @time_range = time_range
+      @week_start = week_start
+      @day_start = day_start
+
+      if relation.default_timezone == :local
+        raise Groupdate::Error, "ActiveRecord::Base.default_timezone must be :utc to use Groupdate"
+      end
+    end
+
+    def generate
+      @relation.group(group_clause).where(*where_clause)
+    end
+
+    private
+
+    def group_clause
+      time_zone = @time_zone.tzinfo.name
+      adapter_name = @relation.connection.adapter_name
+      query =
+        case adapter_name
+        when "SQLServer"
+          sql_server_group_clause(time_zone)
+        when "MySQL", "Mysql2", "Mysql2Spatial", 'Mysql2Rgeo'
+          case period
+          when :day_of_week
+            ["DAYOFWEEK(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?)) - 1", time_zone]
+          when :hour_of_day
+            ["(EXTRACT(HOUR from CONVERT_TZ(#{column}, '+00:00', ?)) + 24 - #{day_start / 3600}) % 24", time_zone]
+          when :minute_of_hour
+            ["(EXTRACT(MINUTE from CONVERT_TZ(#{column}, '+00:00', ?)))", time_zone]
+          when :day_of_month
+            ["DAYOFMONTH(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?))", time_zone]
+          when :month_of_year
+            ["MONTH(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?))", time_zone]
+          when :week
+            ["CONVERT_TZ(DATE_FORMAT(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL ((#{7 - week_start} + WEEKDAY(CONVERT_TZ(#{column}, '+00:00', ?) - INTERVAL #{day_start} second)) % 7) DAY) - INTERVAL #{day_start} second, '+00:00', ?), '%Y-%m-%d 00:00:00') + INTERVAL #{day_start} second, ?, '+00:00')", time_zone, time_zone, time_zone]
+          when :quarter
+            ["DATE_ADD(CONVERT_TZ(DATE_FORMAT(DATE(CONCAT(EXTRACT(YEAR FROM CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?)), '-', LPAD(1 + 3 * (QUARTER(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?)) - 1), 2, '00'), '-01')), '%Y-%m-%d %H:%i:%S'), ?, '+00:00'), INTERVAL #{day_start} second)", time_zone, time_zone, time_zone]
+          else
+            format =
+              case period
+              when :second
+                "%Y-%m-%d %H:%i:%S"
+              when :minute
+                "%Y-%m-%d %H:%i:00"
+              when :hour
+                "%Y-%m-%d %H:00:00"
+              when :day
+                "%Y-%m-%d 00:00:00"
+              when :month
+                "%Y-%m-01 00:00:00"
+              else # year
+                "%Y-01-01 00:00:00"
+              end
+
+            ["DATE_ADD(CONVERT_TZ(DATE_FORMAT(CONVERT_TZ(DATE_SUB(#{column}, INTERVAL #{day_start} second), '+00:00', ?), '#{format}'), ?, '+00:00'), INTERVAL #{day_start} second)", time_zone, time_zone]
+          end
+        when "PostgreSQL", "PostGIS"
+          case period
+          when :day_of_week
+            ["EXTRACT(DOW from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
+          when :hour_of_day
+            ["EXTRACT(HOUR from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
+          when :minute_of_hour
+            ["EXTRACT(MINUTE from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
+          when :day_of_month
+            ["EXTRACT(DAY from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
+          when :month_of_year
+            ["EXTRACT(MONTH from #{column}::timestamptz AT TIME ZONE ? - INTERVAL '#{day_start} second')::integer", time_zone]
+          when :week # start on Sunday, not PostgreSQL default Monday
+            ["(DATE_TRUNC('#{period}', (#{column}::timestamptz - INTERVAL '#{week_start} day' - INTERVAL '#{day_start} second') AT TIME ZONE ?) + INTERVAL '#{week_start} day' + INTERVAL '#{day_start} second') AT TIME ZONE ?", time_zone, time_zone]
+          else
+            ["(DATE_TRUNC('#{period}', (#{column}::timestamptz - INTERVAL '#{day_start} second') AT TIME ZONE ?) + INTERVAL '#{day_start} second') AT TIME ZONE ?", time_zone, time_zone]
+          end
+        when "SQLite"
+          raise Groupdate::Error, "Time zones not supported for SQLite" unless @time_zone.utc_offset.zero?
+          raise Groupdate::Error, "day_start not supported for SQLite" unless day_start.zero?
+          raise Groupdate::Error, "week_start not supported for SQLite" unless week_start == 6
+
+          if period == :week
+            ["strftime('%%Y-%%m-%%d 00:00:00 UTC', #{column}, '-6 days', 'weekday 0')"]
+          else
+            format =
+              case period
+              when :hour_of_day
+                "%H"
+              when :minute_of_hour
+                "%M"
+              when :day_of_week
+                "%w"
+              when :day_of_month
+                "%d"
+              when :month_of_year
+                "%m"
+              when :second
+                "%Y-%m-%d %H:%M:%S UTC"
+              when :minute
+                "%Y-%m-%d %H:%M:00 UTC"
+              when :hour
+                "%Y-%m-%d %H:00:00 UTC"
+              when :day
+                "%Y-%m-%d 00:00:00 UTC"
+              when :month
+                "%Y-%m-01 00:00:00 UTC"
+              when :quarter
+                raise Groupdate::Error, "Quarter not supported for SQLite"
+              else # year
+                "%Y-01-01 00:00:00 UTC"
+              end
+
+            ["strftime('#{format.gsub(/%/, '%%')}', #{column})"]
+          end
+        when "Redshift"
+          case period
+          when :day_of_week
+            ["EXTRACT(DOW from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
+          when :hour_of_day
+            ["EXTRACT(HOUR from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
+          when :minute_of_hour
+            ["EXTRACT(MINUTE from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
+          when :day_of_month
+            ["EXTRACT(DAY from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
+          when :month_of_year
+            ["EXTRACT(MONTH from CONVERT_TIMEZONE(?, #{column}::timestamp) - INTERVAL '#{day_start} second')::integer", time_zone]
+          when :week # start on Sunday, not Redshift default Monday
+            # Redshift does not return timezone information; it
+            # always says it is in UTC time, so we must convert
+            # back to UTC to play properly with the rest of Groupdate.
+            #
+            ["CONVERT_TIMEZONE(?, 'Etc/UTC', DATE_TRUNC(?, CONVERT_TIMEZONE(?, #{column}) - INTERVAL '#{week_start} day' - INTERVAL '#{day_start} second'))::timestamp + INTERVAL '#{week_start} day' + INTERVAL '#{day_start} second'", time_zone, period, time_zone]
+          else
+            ["CONVERT_TIMEZONE(?, 'Etc/UTC', DATE_TRUNC(?, CONVERT_TIMEZONE(?, #{column}) - INTERVAL '#{day_start} second'))::timestamp + INTERVAL '#{day_start} second'", time_zone, period, time_zone]
+          end
+        else
+          raise Groupdate::Error, "Connection adapter not supported: #{adapter_name}"
+        end
+
+      if adapter_name == "MySQL" && period == :week
+        query[0] = "CAST(#{query[0]} AS DATETIME)"
+      end
+
+      clause = @relation.send(:sanitize_sql_array, query)
+
+      # cleaner queries in logs
+      clause = clean_group_clause_postgresql(clause)
+      clean_group_clause_mysql(clause)
+    end
+
+    def clean_group_clause_postgresql(clause)
+      clause.gsub(/ (\-|\+) INTERVAL '0 second'/, "")
+    end
+
+    def clean_group_clause_mysql(clause)
+      clause = clause.gsub("DATE_SUB(#{column}, INTERVAL 0 second)", "#{column}")
+      if clause.start_with?("DATE_ADD(") && clause.end_with?(", INTERVAL 0 second)")
+        clause = clause[9..-21]
+      end
+      clause
+    end
+
+    def where_clause
+      if @time_range.is_a?(Range)
+        op = @time_range.exclude_end? ? "<" : "<="
+        ["#{column} >= ? AND #{column} #{op} ?", @time_range.first, @time_range.last]
+      else
+        ["#{column} IS NOT NULL"]
+      end
+    end
+
+    # resolves eagerly
+    # need to convert both where_clause (easy)
+    # and group_clause (not easy) if want to avoid this
+    def resolve_column(relation, column)
+      node = relation.send(:relation).send(:arel_columns, [column]).first
+      node = Arel::Nodes::SqlLiteral.new(node) if node.is_a?(String)
+      relation.connection.visitor.accept(node, Arel::Collectors::SQLString.new).value
+    end
+  end
+end

data/lib/groupdate/series_builder.rb

@@ -0,0 +1,264 @@
+module Groupdate
+  class SeriesBuilder
+    attr_reader :period, :time_zone, :day_start, :week_start, :options
+
+    CHECK_PERIODS = [:day, :week, :month, :quarter, :year]
+
+    def initialize(period:, time_zone:, day_start:, week_start:, **options)
+      @period = period
+      @time_zone = time_zone
+      @week_start = week_start
+      @day_start = day_start
+      @options = options
+      @round_time = {}
+    end
+
+    def generate(data, default_value:, series_default: true, multiple_groups: false, group_index: nil)
+      series = generate_series(data, multiple_groups, group_index)
+      series = handle_multiple(data, series, multiple_groups, group_index)
+
+      unless entire_series?(series_default)
+        series = series.select { |k| data[k] }
+      end
+
+      value = 0
+      result = Hash[series.map do |k|
+        value = data.delete(k) || (@options[:carry_forward] && value) || default_value
+        key =
+          if multiple_groups
+            k[0...group_index] + [key_format.call(k[group_index])] + k[(group_index + 1)..-1]
+          else
+            key_format.call(k)
+          end
+
+        [key, value]
+      end]
+
+      # only check for database
+      # only checks remaining keys to avoid expensive calls to round_time
+      if series_default && CHECK_PERIODS.include?(period)
+        check_consistent_time_zone_info(data, multiple_groups, group_index)
+      end
+
+      result
+    end
+
+    def round_time(time)
+      time = time.to_time.in_time_zone(time_zone)
+
+      # only if day_start != 0 for performance
+      time -= day_start.seconds if day_start != 0
+
+      time =
+        case period
+        when :second
+          time.change(usec: 0)
+        when :minute
+          time.change(sec: 0)
+        when :hour
+          time.change(min: 0)
+        when :day
+          time.beginning_of_day
+        when :week
+          # same logic as MySQL group
+          weekday = (time.wday - 1) % 7
+          (time - ((7 - week_start + weekday) % 7).days).midnight
+        when :month
+          time.beginning_of_month
+        when :quarter
+          time.beginning_of_quarter
+        when :year
+          time.beginning_of_year
+        when :hour_of_day
+          time.hour
+        when :minute_of_hour
+          time.min
+        when :day_of_week
+          (time.wday - 1 - week_start) % 7
+        when :day_of_month
+          time.day
+        when :month_of_year
+          time.month
+        else
+          raise Groupdate::Error, "Invalid period"
+        end
+
+      # only if day_start != 0 for performance
+      time += day_start.seconds if day_start != 0 && time.is_a?(Time)
+
+      time
+    end
+
+    def time_range
+      @time_range ||= begin
+        time_range = options[:range]
+        if time_range.is_a?(Range) && time_range.first.is_a?(Date)
+          # convert range of dates to range of times
+          # use parsing instead of in_time_zone due to Rails < 4
+          last = time_zone.parse(time_range.last.to_s)
+          last += 1.day unless time_range.exclude_end?
+          time_range = Range.new(time_zone.parse(time_range.first.to_s), last, true)
+        elsif !time_range && options[:last]
+          if period == :quarter
+            step = 3.months
+          elsif 1.respond_to?(period)
+            step = 1.send(period)
+          else
+            raise ArgumentError, "Cannot use last option with #{period}"
+          end
+          if step
+            # loop instead of multiply to change start_at - see #151
+            start_at = now
+            (options[:last].to_i - 1).times do
+              start_at -= step
+            end
+
+            time_range =
+              if options[:current] == false
+                round_time(start_at - step)...round_time(now)
+              else
+                round_time(start_at)..now
+              end
+          end
+        end
+        time_range
+      end
+    end
+
+    private
+
+    def now
+      @now ||= time_zone.now
+    end
+
+    def generate_series(data, multiple_groups, group_index)
+      case period
+      when :day_of_week
+        0..6
+      when :hour_of_day
+        0..23
+      when :minute_of_hour
+        0..59
+      when :day_of_month
+        1..31
+      when :month_of_year
+        1..12
+      else
+        time_range = self.time_range
+        time_range =
+          if time_range.is_a?(Range)
+            time_range
+          else
+            # use first and last values
+            sorted_keys =
+              if multiple_groups
+                data.keys.map { |k| k[group_index] }.sort
+              else
+                data.keys.sort
+              end
+
+            tr = sorted_keys.first..sorted_keys.last
+            if options[:current] == false && sorted_keys.any? && round_time(now) >= tr.last
+              tr = tr.first...round_time(now)
+            end
+            tr
+          end
+
+        if time_range.first
+          series = [round_time(time_range.first)]
+
+          if period == :quarter
+            step = 3.months
+          else
+            step = 1.send(period)
+          end
+
+          last_step = series.last
+          loop do
+            next_step = last_step + step
+            next_step = round_time(next_step) if next_step.hour != day_start # add condition to speed up
+            break unless time_range.cover?(next_step)
+
+            if next_step == last_step
+              last_step += step
+              next
+            end
+            series << next_step
+            last_step = next_step
+          end
+
+          series
+        else
+          []
+        end
+      end
+    end
+
+    def key_format
+      locale = options[:locale] || I18n.locale
+      use_dates = options.key?(:dates) ? options[:dates] : Groupdate.dates
+
+      if options[:format]
+        if options[:format].respond_to?(:call)
+          options[:format]
+        else
+          sunday = time_zone.parse("2014-03-02 00:00:00")
+          lambda do |key|
+            case period
+            when :hour_of_day
+              key = sunday + key.hours + day_start.seconds
+            when :minute_of_hour
+              key = sunday + key.minutes + day_start.seconds
+            when :day_of_week
+              key = sunday + key.days + (week_start + 1).days
+            when :day_of_month
+              key = Date.new(2014, 1, key).to_time
+            when :month_of_year
+              key = Date.new(2014, key, 1).to_time
+            end
+            I18n.localize(key, format: options[:format], locale: locale)
+          end
+        end
+      elsif [:day, :week, :month, :quarter, :year].include?(period) && use_dates
+        lambda { |k| k.to_date }
+      else
+        lambda { |k| k }
+      end
+    end
+
+    def handle_multiple(data, series, multiple_groups, group_index)
+      reverse = options[:reverse]
+
+      if multiple_groups
+        keys = data.keys.map { |k| k[0...group_index] + k[(group_index + 1)..-1] }.uniq
+        series = series.to_a.reverse if reverse
+        keys.flat_map do |k|
+          series.map { |s| k[0...group_index] + [s] + k[group_index..-1] }
+        end
+      elsif reverse
+        series.to_a.reverse
+      else
+        series
+      end
+    end
+
+    def check_consistent_time_zone_info(data, multiple_groups, group_index)
+      keys = data.keys
+      if multiple_groups
+        keys.map! { |k| k[group_index] }
+        keys.uniq!
+      end
+
+      keys.each do |key|
+        if key != round_time(key)
+          # only need to show what database returned since it will cast in Ruby time zone
+          raise Groupdate::Error, "Database and Ruby have inconsistent time zone info. Database returned #{key} != #{round_time(key)}"
+        end
+      end
+    end
+
+    def entire_series?(series_default)
+      options.key?(:series) ? options[:series] : series_default
+    end
+  end
+end