martyr 0.1.74.pre

data/lib/martyr/runtime/pivot/pivot_table.rb

@@ -0,0 +1,109 @@
+module Martyr
+  module Runtime
+    class PivotTable
+      include ActiveModel::Model
+
+      # @attribute metrics [Array<BaseMetric>]
+      # @attribute row_axis [PivotAxis]
+      # @attribute column_axis [PivotAxis]
+      # @attribute pivot_grain [Array<String>] array of level ids
+      attr_accessor :metrics, :row_axis, :column_axis, :pivot_grain, :row_totals, :column_totals, :sort
+      attr_reader :elements
+
+      def initialize(query_context, *args)
+        super(*args)
+
+        # We don't restrict metrics since custom rollups may have dependencies
+        @elements = query_context.elements(levels: pivot_grain, sort: sort)
+      end
+
+      def reload
+        @_cells = nil
+        @_lowest_cells = nil
+        row_axis.load_values(cells, reset: true)
+        column_axis.load_values(cells, reset: true)
+        self
+      end
+
+      def inspect
+        {metrics: metrics.map(&:id), levels: pivot_grain, on_rows: row_axis, on_columns: column_axis, totals: {rows: row_totals, columns: column_totals}}.inspect
+      end
+
+      def cells
+        @_cells ||= sort_cells(lowest_cells + sub_totals)
+      end
+
+      def lowest_cells
+        @_lowest_cells ||= metrics.flat_map do |metric|
+          elements.map do |element|
+            PivotCell.new metric, element
+          end
+        end
+      end
+
+      def sub_totals
+        resettable_grain = row_totals ? row_axis.ids : []
+        resettable_grain += column_totals ? column_axis.ids : []
+        resettable_grain -= [:metrics]
+        (0...resettable_grain.length).flat_map do |x|
+          reset = resettable_grain[x..-1]
+          elements.index_by do |element|
+            (pivot_grain - reset).map{|level_id| element[level_id]}
+          end.flat_map do |_sub_total_key, representative|
+            element = representative.locate reset: reset
+            metrics.map do |metric|
+              PivotCell.new(metric, element, reset)
+            end
+          end.reject(:empty?).compact
+        end
+      end
+
+      def transpose
+        self.row_axis, self.column_axis = column_axis, row_axis
+        reload
+      end
+
+      def to_chart(name: nil)
+        cells.group_by do |cell|
+          cell.to_axis_values(row_axis)
+        end.map do |row_grain_values, cells|
+          data = cells.inject(column_axis.flat_values_nil_hash) { |h, cell| h.merge! cell.to_axis_values(column_axis) => cell.value }
+          {name: name || row_grain_values, data: data}
+        end
+      end
+
+      def to_csv
+        CSV.generate do |csv|
+          column_axis.add_header_column_cells_to_csv(csv, row_axis)
+          prev_row = nil
+          rows.each do |row|
+            csv << prev_row = row.to_a(previous: prev_row)
+          end
+        end
+      end
+
+      def rows
+        cells.group_by do |cell|
+          cell.to_axis_values(row_axis, flat: false)
+        end.map do |row_grain_values, cells|
+          PivotRow.new(self, row_grain_values, cells)
+        end
+      end
+
+      private
+
+      def metrics_sort_order
+        @_metrics_sort_order ||= Hash[metrics.each_with_index.map{|m, i| [m.id, i]}]
+      end
+
+      def sort_cells(cells_arr)
+        return cells_arr if sort.present?
+        cells_arr.sort_by do |cell|
+          pivot_grain.map{|level_id| cell[level_id] || '' } + [metrics_sort_order[cell.metric_id]]
+        end
+      end
+
+    end
+  end
+end
+

data/lib/martyr/runtime/pivot/pivot_table_builder.rb

@@ -0,0 +1,125 @@
+module Martyr
+  module Runtime
+    class PivotTableBuilder
+      ALL_METRICS_KEY = 'metrics'
+
+      attr_reader :query_context, :on_columns_args, :on_rows_args, :sort_args, :in_cells_arg, :row_totals, :column_totals
+      delegate :standardizer, :definition_from_id, to: :query_context
+
+      delegate :cells, :elements, :to_chart, :to_csv, to: :table
+
+      def initialize(query_context)
+        @query_context = query_context
+        @on_columns_args = []
+        @on_rows_args = []
+        @sort_args = {}
+        @row_totals = false
+        @column_totals = false
+      end
+
+      def select(*metric_ids)
+        dup.instance_eval do
+          @metrics = standardizer.standardize(metric_ids)
+          self
+        end
+      end
+
+      def sort(*args)
+        @sort_args.merge! Sorter.args_to_hash(args.length == 1 ? args.first : args)
+        self
+      end
+
+      def on_columns(*level_ids)
+        dup.instance_eval do
+          validate_metrics_in_level_ids(level_ids)
+          @on_columns_args = level_ids.uniq
+          self
+        end
+      end
+
+      def on_rows(*level_ids)
+        dup.instance_eval do
+          validate_metrics_in_level_ids(level_ids)
+          @on_rows_args = level_ids.uniq
+          self
+        end
+      end
+
+      def with_totals(rows: true, columns: true)
+        dup.instance_eval do
+          @row_totals = !!rows
+          @column_totals = !!columns
+          self
+        end
+      end
+
+      def in_cells(metric_id)
+        raise Query::Error.new('Cannot have more than one metric in cells') if metric_id.to_s == ALL_METRICS_KEY
+        raise Query::Error.new("#{metric_id} is not a metric") unless metric?(metric_id)
+        @in_cells_arg = standardizer.standardize(metric_id)
+        self
+      end
+
+      def build
+        raise Query::Error.new('No columns were selected') unless on_columns_args.present?
+        raise Query::Error.new('No rows were selected') unless on_rows_args.present?
+        raise Query::Error.new('At least one metric has to be defined in pivot') unless metric_definition_count > 0
+        raise Query::Error.new('Metrics can either be on columns or rows or in cells') if metric_definition_count > 1
+        PivotTable.new(query_context, metrics: metrics, pivot_grain: pivot_grain,
+                       row_axis: axis_for(on_rows_args), column_axis: axis_for(on_columns_args),
+                       row_totals: row_totals, column_totals: column_totals, sort: sort_args).reload
+      end
+      alias_method :table, :build
+
+      private
+
+      def metric?(id)
+        return true if id.to_s == ALL_METRICS_KEY
+        query_context.metric? standardizer.standardize(id)
+      end
+
+      def validate_metrics_in_level_ids(level_ids)
+        level_ids.each do |level_id|
+          next if level_id.to_s == ALL_METRICS_KEY
+          level_id = standardizer.standardize(level_id)
+          raise Query::Error.new("#{level_id}: Cannot pivot on individual metrics. " +
+                                     "Use 'metrics' for `on_columns` or `on_rows`, " +
+                                     "`select` to restrict metrics or `in_cells` to display one metric") if metric?(level_id)
+        end
+      end
+
+      def metric_ids
+        return [in_cells_arg] if in_cells_arg
+        @metrics || query_context.metric_ids
+      end
+
+      def metric_definition_count
+        (in_cells_arg ? 1 : 0) + (on_columns_args + on_rows_args).select { |x| metric?(x) }.length
+      end
+
+      def without_metrics(collection)
+        collection.reject { |x| metric?(x) }
+      end
+
+      def metrics
+        metric_ids.map { |x| query_context.metric(x) }
+      end
+
+      def pivot_grain
+        without_metrics(on_columns_args + on_rows_args)
+      end
+
+      def axis_for(collection)
+        grain_elements = collection.map do |level_id|
+          if level_id.to_s == ALL_METRICS_KEY
+            Runtime::PivotGrainElement.new(id: level_id, metrics: metrics)
+          else
+            Runtime::PivotGrainElement.new(id: level_id, level_definition: query_context.definition_from_id(level_id))
+          end
+        end
+        PivotAxis.new grain_elements: grain_elements
+      end
+
+    end
+  end
+end

data/lib/martyr/runtime/query/metric_dependency_resolver.rb

@@ -0,0 +1,149 @@
+module Martyr
+  module Runtime
+    class MetricDependencyResolver
+      attr_reader :cube
+
+      # @param cube [BaseCube] either virtual or regular cube
+      def initialize(cube)
+        @cube = cube
+        @metrics_by_cube = {}
+        @inferred_fact_grain_by_cube = {}
+      end
+
+      def to_hash
+        Hash[@metrics_by_cube.map{ |cube_name, arr| [cube_name, arr.keys] }]
+      end
+
+      def inspect
+        to_hash.inspect
+      end
+
+      # @option all [Boolean] send true if all metrics, including dependents, should be retrieved. Otherwise, only
+      #   explicitly asked-for metrics will be included in the result set
+      # @return [Array<BaseMetric>]
+      def metrics(all: false)
+        metric_entries = @metrics_by_cube.flat_map { |_cube_name, metric_ids_hash| metric_ids_hash.values }
+        metric_entries.select! { |entry| entry[:explicit] } unless all
+        metric_entries.map { |entry| entry[:metric] }
+      end
+
+      # @return [Array<String>] metric IDs
+      def metric_ids(all: false)
+        metrics(all: all).map(&:id)
+      end
+
+      # @param cube_name [String]
+      # @option all [Boolean] send true if all metrics, including dependents, should be retrieved. Otherwise, only
+      #   explicitly asked-for metrics will be included in the result set
+      # @return [Array<BaseMetric>]
+      def metrics_for(cube_name, all: false)
+        relevant_entries_for(cube_name, all: all).map do |_metric_id, metric_entry|
+          metric_entry[:metric]
+        end
+      end
+
+      # @see metrics_for
+      # @return [Array<String>] metric IDs
+      def metric_ids_for(cube_name, all: false)
+        relevant_entries_for(cube_name, all: all).map(&:first)
+      end
+
+      # @return [Array<String>] of all fact grains combined acrross cubes
+      def inferred_fact_grain
+        @inferred_fact_grain_by_cube.flat_map{ |_cube_name, levels_lookup_hash| levels_lookup_hash.keys }.uniq
+      end
+
+      # @return [Array<String>] array of level IDs that need to be part of the fact grain in order for the metrics to
+      #   compute, including any dependent metrics. This does not include the default_fact_grain
+      def inferred_fact_grain_for(cube_name)
+        @inferred_fact_grain_by_cube[cube_name].try(:keys) || []
+      end
+
+      # @return [Array<String>] array of sub facts that need to be joined in order to support the required metrics,
+      #   as provided under the `sub_queries` key of the metric definition.
+      def sub_facts_for(cube_name)
+        metrics_for(cube_name, all: true).flat_map {|metric| metric.sub_queries if metric.respond_to?(:sub_queries) }.compact
+      end
+
+      # Recursively add the metric and its dependencies
+      # @param metric_id [String] fully qualified metric ID (with cube name)
+      # @option explicit [Boolean] indicates whether the metric was asked to be included as part of a query.
+      #   send false for metrics that were added due to dependency.
+      def add_metric(metric_id, explicit: true)
+        metric = cube.find_metric_id(metric_id)
+        add_count_distinct_fact_grain_dependency(metric, explicit)
+        return unless register_metric(metric, explicit)
+
+        add_fact_grain_dependency(metric)
+        add_dependent_metrics(metric)
+      end
+
+      def data_dup
+        dup.instance_eval do
+          @metrics_by_cube = @metrics_by_cube.deep_dup
+          @inferred_fact_grain_by_cube = @inferred_fact_grain_by_cube.deep_dup
+          self
+        end
+      end
+
+      private
+
+      # @param metric [BaseMetric]
+      # @param explicit [Boolean] see #add_metric
+      # @return [Boolean]
+      #   true if the metric was added for the first time.
+      #   false if the metric was already added before
+      #
+      # @note this method makes sure to set the explicit flag to true if explicit param is true. The flag will not be
+      #   changed if explicit param is false.
+      #
+      # This makes sure that if the user specifies select('a', 'b') and 'a' depends on 'b', then 'b' will be marked
+      # as explicit, despite the fact it was added as explicit=false when 'a' was added.
+      #
+      def register_metric(metric, explicit)
+        @metrics_by_cube[metric.cube_name] ||= {}
+        if @metrics_by_cube[metric.cube_name].has_key?(metric.id)
+          @metrics_by_cube[metric.cube_name][metric.id][:explicit] = true if explicit
+          return false
+        end
+
+        @metrics_by_cube[metric.cube_name][metric.id] = { metric: metric, explicit: explicit }
+        true
+      end
+
+      # The level which count-distinct metric A depends on is added only if another metric depends on metric A,
+      # and only if the user did not specify a custom fact_grain for metric A.
+      def add_count_distinct_fact_grain_dependency(metric, explicit)
+        return unless !explicit and metric.is_a?(Schema::CountDistinctMetric) and metric.fact_grain.blank?
+        store_inferred_fact_grain(metric.cube_name, metric.level_id)
+      end
+
+      def add_fact_grain_dependency(metric)
+        return unless metric.respond_to?(:fact_grain) and metric.fact_grain.present?
+        metric.fact_grain.each do |level_id|
+          store_inferred_fact_grain(metric.cube_name, level_id)
+        end
+      end
+
+      # @param metric [BaseMetric]
+      def add_dependent_metrics(metric)
+        return unless metric.respond_to?(:depends_on) and metric.depends_on.present?
+        metric.depends_on.each do |dependent_metric_id|
+          add_metric(dependent_metric_id, explicit: false)
+        end
+      end
+
+      # @see metrics_for
+      def relevant_entries_for(cube_name, all:)
+        candidates = @metrics_by_cube[cube_name] || []
+        candidates.select!{ |_metric_id, metric_entry| metric_entry[:explicit] } unless all
+        candidates
+      end
+
+      def store_inferred_fact_grain(cube_name, level_id)
+        @inferred_fact_grain_by_cube[cube_name] ||= {}
+        @inferred_fact_grain_by_cube[cube_name][level_id] = true
+      end
+    end
+  end
+end

data/lib/martyr/runtime/query/query_context.rb

@@ -0,0 +1,246 @@
+module Martyr
+  module Runtime
+    class QueryContext
+      include Martyr::LevelComparator
+      include Martyr::Translations
+
+      # @attribute metrics [Array<BaseMetric>] metrics that were requested as part of the query, without dependencies
+      # @attribute sub_cubes_hash [Hash] of the format { cube_name => Runtime::SubCube }
+      # @attribute dimension_scopes [Runtime::DimensionScopeCollection] see BaseCube::build_dimension_scopes
+      # @attribute level_ids_in_grain [Array<String>] array of level IDs
+      # @attribute virtual_cube [VirtualCube]
+      # @attribute virtual_cube_metric_ids [Array<String>]
+
+      attr_accessor :metrics, :sub_cubes_hash, :dimension_scopes, :level_ids_in_grain, :virtual_cube, :virtual_cube_metric_ids
+      attr_reader :data_slice
+      delegate :level_scope, :level_scopes, :with_level_scope, :lowest_level_of, :lowest_level_ids_of,
+        :levels_and_above_for, :level_ids_and_above_for, :level_loaded?, to: :dimension_scopes
+      delegate :slice, to: :memory_slice
+
+      def initialize
+        @data_slice = DataSlice.new(self)
+        @sub_cubes_hash = {}
+        @virtual_cube_metric_ids = []
+      end
+
+      def inspect
+        "#<QueryContext metric_ids: #{metric_ids}, grain: #{level_ids_in_grain}, memory_slice: #{memory_slice.to_hash}, data_slice: #{data_slice.to_hash}, sub_cubes: #{sub_cubes}>"
+      end
+
+      def sub_cubes
+        sub_cubes_hash.values
+      end
+
+      # @return [Array<BaseMetric>] if the current cube is virtual, returns array of metrics of the virtual cube
+      def virtual_metrics
+        return [] unless self.virtual_cube
+        virtual_cube_metric_ids.map do |unique_metric_id|
+          metric_id = second_element_from_id(unique_metric_id)
+          virtual_cube.metric_definitions.find_or_error(metric_id)
+        end
+      end
+
+      # @return [Array<String>] only metric IDs that were requested as part of the query, without dependencies
+      def metric_ids
+        metrics.map(&:id)
+      end
+
+      # @return [Array<BaseMetric>] all metrics, including those that are added by dependencies and virtuals
+      def all_metrics
+        sub_cubes.flat_map { |sub_cube| sub_cube.metric_objects } + virtual_metrics
+      end
+
+      def all_metric_ids
+        all_metrics.map(&:id)
+      end
+
+      # @param id [String] has to be fully qualified (cube_name.metric_name)
+      def metric(id)
+        metric_ids_lookup[id]
+      end
+
+      # @param id [String] has to be fully qualified (cube_name.metric_name)
+      def metric?(id)
+        !!metric(id)
+      end
+
+      # = Grain
+
+      def supported_level_ids
+        @_supported_level_ids ||= levels_and_above_for(level_ids_in_grain).map(&:id)
+      end
+
+      def validate_slice_on!(slice_on)
+        slice_on_object = definition_from_id(slice_on)
+        raise Query::Error.new("Cannot find `#{slice_on}`") unless slice_on_object
+        raise Query::Error.new("Cannot slice on `#{slice_on}`: it is not in the grain") if slice_on_object.is_a?(Martyr::Level) and !supported_level_ids.include?(slice_on)
+        true
+      end
+
+      # = Memory slices
+
+      def memory_slice
+        @memory_slice ||= MemorySlice.new(data_slice)
+      end
+
+      # @return [QueryContext] for chaining
+      def slice(*args)
+        dup_internals.slice!(*args)
+      end
+
+      # @return [QueryContext] for chaining
+      def slice!(*args)
+        memory_slice.slice(*args)
+        self
+      end
+
+      # @return [Array<String>] array of level IDs that are in the memory slice
+      def sliced_level_ids
+        memory_slice.keys.reject{|id| metric?(id)}
+      end
+
+      # @return [Array<String>] array of level IDs that are in the grain but not sliced
+      def unsliced_level_ids_in_grain
+        level_ids_in_grain - sliced_level_ids
+      end
+
+      # = Run
+
+      # @option cube_name [String, Symbol] default is first cube 
+      # @return [Array<Fact>] of the chosen cube
+      def facts(cube_name = nil)
+        cube_name ||= default_cube.cube_name
+        sub_cubes_hash[cube_name.to_s].facts
+      end
+
+      # A cube that has no grain and no metric doesn't matter - it will end up having one "useless" element with
+      # no levels in the grain.
+      # TODO: ignore cubes that do not share a grain or slice. Here is an algorithm:
+      #   Start with a set of all metrics that are needed to be fetched.
+      #   Add all cubes with metric-slices on them.
+      #   If the shared grain is missing a level in the grain - add all cubes that support that level.
+      #   If the shared grain is missing a level in ths slice - add all cubes that support that level.
+      #
+      # @option sort [Array, Hash] either
+      def elements(**options)
+        load_bottom_level_primary_keys
+        builder = VirtualElementsBuilder.new(memory_slice, unsliced_level_ids_in_grain: unsliced_level_ids_in_grain,
+          virtual_metrics: virtual_metrics)
+
+        sort_args = options.delete(:sort) || {}
+        sorter = Sorter.new(standardizer.standardize(sort_args)) { |sort_argument| definition_from_id(sort_argument) }
+
+        sub_cubes.each do |sub_cube|
+          next unless sub_cube.metric_objects.present? or sub_cube.lowest_level_ids_in_grain.present?
+          memory_slice_for_cube = memory_slice.for_cube(sub_cube)
+          builder.add sub_cube.elements(memory_slice_for_cube, **options), sliced: memory_slice_for_cube.to_hash.present?
+        end
+        sorter.sort(builder.build)
+      end
+
+      def total(metrics: nil)
+        elements(levels: [], metrics: metrics).first || empty_element(metrics: metrics)
+      end
+      alias_method :totals, :total
+
+      def empty_element(metrics: nil)
+        return sub_cubes.first.element_locator_for(memory_slice, metrics: metrics).empty_element unless virtual_cube?
+
+        locators = sub_cubes.map {|sub_cube| sub_cube.element_locator_for(memory_slice, metrics: metrics) }
+        VirtualElement.new({}, memory_slice, locators, []).rollup(*virtual_metrics)
+      end
+
+      def pivot
+        Runtime::PivotTableBuilder.new(self)
+      end
+
+      # = Dispatcher
+
+      # @return [BaseMetric, DimensionReference, BaseLevelDefinition]
+      def definition_from_id(id)
+        with_standard_id(id) do |x, y|
+          return dimension_scopes[x].try(:dimension_definition) || default_cube.metrics[x] if !y
+          return sub_cubes_hash[x].find_metric(y) if sub_cubes_hash[x]
+          dimension_scopes.find_level(id).try(:level_definition)
+        end
+      end
+
+      # = As Dimension Bus Role
+
+      def level_ids_and_above
+        level_ids_and_above_for(level_ids_in_grain)
+      end
+
+      # @param level_id [String] e.g. 'customers.last_name'
+      # @param fact_record [Fact]
+      def fetch_unsupported_level_value(level_id, fact_record)
+        sought_level_definition = dimension_scopes.find_level(level_id).level_definition
+        common_denominator_association = fact_record.sub_cube.common_denominator_level_association(level_id, prefer_query: true)
+        common_denominator_level_scope = level_scope(common_denominator_association.id)
+        common_denominator_level_scope.recursive_lookup_up fact_record.fact_key_for(common_denominator_association.id), level: sought_level_definition
+      end
+
+      # @param level_id [String] e.g. 'customers.last_name'
+      # @param fact_key_value [Integer] the primary key stored in the fact
+      def fetch_supported_query_level_record(level_id, fact_key_value)
+        level_scope = level_scope(level_id)
+        raise Internal::Error.new('level must be query') unless level_scope.query?
+        level_scope.recursive_lookup_up fact_key_value, level: level_scope
+      end
+
+      def standardizer
+        @standardizer ||= Martyr::MetricIdStandardizer.new(default_cube.cube_name, raise_if_not_ok: virtual_cube?)
+      end
+
+      def dup_internals
+        dup.instance_eval do
+          @memory_slice = memory_slice.dup_internals
+          self
+        end
+      end
+
+      def element_helper_module
+        return @element_helper_module if @element_helper_module
+        @element_helper_module = Module.new
+        dimension_scopes.register_element_helper_methods(@element_helper_module)
+
+        all_metric_ids.each do |metric_id|
+          metric_name = second_element_from_id(metric_id)
+          @element_helper_module.module_eval do
+            define_method(metric_name) { fetch(metric_id) }
+          end
+        end
+        @element_helper_module
+      end
+
+      private
+
+      def metric_ids_lookup
+        @metric_ids_lookup ||= all_metrics.index_by(&:id)
+      end
+
+      def virtual_cube?
+        sub_cubes.length > 1
+      end
+
+      def default_cube
+        sub_cubes.first
+      end
+
+      def load_bottom_level_primary_keys
+        return if @bottom_level_primary_keys_loaded
+        sub_cubes.each do |sub_cube|
+          sub_cube.lowest_level_ids_in_grain.each do |level_id|
+            level = level_scope(level_id)
+            next unless level.query?
+            level.primary_keys_for_load ||= []
+            level.primary_keys_for_load += sub_cube.facts.map{|x| x.raw[level.fact_alias]}
+            level.primary_keys_for_load = level.primary_keys_for_load.uniq
+          end
+        end
+        @bottom_level_primary_keys_loaded = true
+      end
+
+    end
+  end
+end