martyr 0.1.74.pre

data/lib/martyr/runtime/slices/slice_definitions/metric_slice_definition.rb

@@ -0,0 +1,120 @@
+module Martyr
+  # The conversion from and to interval sets is used for merging between data slices and memory slices.
+  class MetricSliceDefinition < BaseSliceDefinition
+
+    OPERATORS = [:gt, :lt, :gte, :lte, :eq, :not]
+    attr_accessor :metric, *OPERATORS
+
+    def self.from_interval_set(interval_set)
+      base = {metric: metric}
+      new base.merge!(interval_set_to_hash(interval_set))
+    end
+
+    def to_hash
+      {metric.id => OPERATORS.inject({}) { |h, op| send(op) ? h.merge!(op => send(op)) : h }}
+    end
+
+    def merge(other)
+      self.class.from_interval_set(to_interval_set.intersect(other.to_interval_set))
+    end
+
+    # @return [Array<Array<Hash>>]
+    #   Every element in the top level array is an array with "OR" statements.
+    #   The top level statements are to be combined with an "AND".
+    #
+    #   Example:
+    #     [ [1,2,3], [4], [5] ]
+    #     # => "(1 OR 2 OR 3) AND (4) AND (5)"
+    #
+    def combined_statements
+      statements = []
+      statements << [{data_operator: gt_operator, memory_operator: gt_operator, value: gt_value}] if gt_operator
+      statements << [{data_operator: lt_operator, memory_operator: lt_operator, value: lt_value}] if lt_operator
+      statements << Array.wrap(eq).map {|value| {data_operator: '=', memory_operator: '==', value: value} } if eq
+
+      Array.wrap(self.not).each do |value|
+        statements << [{data_operator: '!=', memory_operator: '!=', value: value}]
+      end
+
+      statements
+    end
+
+    protected
+
+    def compile_operators
+      hash = interval_set_to_hash(to_interval_set)
+      OPERATORS.each do |operator|
+        instance_variable_set "@#{operator}", hash[operator]
+      end
+      set_null if hash.compact.empty?
+    end
+
+    def self.interval_set_to_hash(interval_set)
+      return {} if interval_set.null?
+      not_arr = interval_set.extract_and_fill_holes.presence
+      eq_arr = interval_set.extract_and_remove_points.presence
+      raise Internal::Error.new('Unexpected interval set format') unless interval_set.null? or interval_set.continuous?
+
+      upper_point = interval_set.upper_bound
+      lte = upper_point.x if upper_point.try(:closed?)
+      lt = upper_point.x if upper_point.try(:open?)
+
+      lower_point = interval_set.lower_bound
+      gte = lower_point.x if lower_point.try(:closed?)
+      gt = lower_point.x if lower_point.try(:open?)
+
+      { not: not_arr, eq: eq_arr, lte: lte, lt: lt, gte: gte, gt: gt }
+    end
+    delegate :interval_set_to_hash, to: 'self.class'
+
+    # Calculate each time to avoid messing up internal state
+    def to_interval_set
+      interval_set = IntervalSet.new.add
+      merge_eq_interval_set(interval_set)
+      merge_not_interval_set(interval_set)
+      interval_set.intersect IntervalSet.new(to: lt) if lt
+      interval_set.intersect IntervalSet.new(to: [lte]) if lte
+      interval_set.intersect IntervalSet.new(from: gt) if gt
+      interval_set.intersect IntervalSet.new(from: [gte]) if gte
+      interval_set
+    end
+
+    def merge_eq_interval_set(interval_set)
+      return unless eq.present?
+      set = IntervalSet.new
+      Array.wrap(eq).each {|x| set.add(from: [x], to: [x]) }
+      interval_set.intersect(set)
+    end
+
+    def merge_not_interval_set(interval_set)
+      return unless self.not.present?
+      Array.wrap(self.not).
+        map {|x| IntervalSet.new(to: x).add(from: x) }.
+        inject(interval_set) {|set, hole| set.intersect(hole)}
+    end
+
+    def gt_value
+      gte || gt
+    end
+
+    def lt_value
+      lte || lt
+    end
+
+    def gt_operator
+      if gte.present?
+        '>='
+      elsif gt.present?
+        '>'
+      end
+    end
+
+    def lt_operator
+      if lte.present?
+        '<='
+      elsif lt.present?
+        '<'
+      end
+    end
+  end
+end

data/lib/martyr/runtime/slices/slice_definitions/plain_dimension_level_slice_definition.rb

@@ -0,0 +1,26 @@
+module Martyr
+  class PlainDimensionLevelSliceDefinition < BaseSliceDefinition
+
+    # @attribute level [BaseLevelDefinition]
+    attr_accessor :level, :with
+
+    def to_hash
+      {level.id => {with: with}}
+    end
+
+    def merge(other)
+      raise Internal::Error.new('Cannot merge two different levels') unless level.id == other.level.id
+      merged_with = with.present? && other.with.present? ? with & other.with : with + other.with
+      self.class.new(level: level, with: merged_with)
+    end
+
+    private
+
+    def compile_operators
+      @with = Array.wrap(@with).uniq
+      set_null unless @with.present?
+    end
+
+  end
+end
+

data/lib/martyr/runtime/sub_cubes/fact_filler_strategies.rb

@@ -0,0 +1,61 @@
+module Martyr
+  module Runtime
+    module FactFillerStrategies
+
+      # Instead of every fact object figuring out itself how to extract fact values from the raw data,
+      # we calculate a memoized set of strategies in advance.
+
+      def fact_levels_filler_hash
+        return @fact_levels_filler_hash if @fact_levels_filler_hash
+        hash = {}
+        supported_level_definitions.each do |level_definition|
+          level_id = level_definition.id
+          if has_association_with_level?(level_id)
+            level_association = association_from_id(level_id)
+            if level_association.degenerate?
+              filler = DegenerateLevelAssociationFillerStrategy.new(level_association)
+            else
+              filler = QueryLevelAssociationFillerStrategy.new(level_association)
+            end
+          else
+            filler = UnassociatedLevelFillerStrategy.new(level_definition)
+          end
+          hash[level_id] = filler
+        end
+        @fact_levels_filler_hash = hash
+      end
+
+      class DegenerateLevelAssociationFillerStrategy
+        def initialize(level_association)
+          @fact_alias = level_association.fact_alias
+        end
+
+        def value(fact)
+          fact.raw.fetch(@fact_alias)
+        end
+      end
+
+      class QueryLevelAssociationFillerStrategy
+        def initialize(level_association)
+          @level_association = level_association
+        end
+
+        def value(fact)
+          fact_key_value = fact.raw.fetch(@level_association.fact_alias)
+          FutureFactValue.new(fact, @level_association.level_definition, key_supported: true, fact_key_value: fact_key_value)
+        end
+      end
+
+      class UnassociatedLevelFillerStrategy
+        def initialize(level_definition)
+          @level_definition = level_definition
+        end
+
+        def value(fact)
+          FutureFactValue.new(fact, @level_definition, key_supported: false)
+        end
+      end
+
+    end
+  end
+end

data/lib/martyr/runtime/sub_cubes/query_metrics.rb

@@ -0,0 +1,56 @@
+module Martyr
+  module Runtime
+    class QueryMetrics < HashWithIndifferentAccess
+      include Martyr::Registrable
+      include Martyr::Translations
+
+      attr_reader :sub_cube
+
+      delegate :cube, to: :sub_cube
+
+      include Martyr::Delegators
+      each_child_delegator :add_to_select, to: :values
+
+      alias_method :find_metric, :find_or_error
+
+      def initialize(sub_cube)
+        @sub_cube = sub_cube
+      end
+
+      def inspect
+        "#<#{self.class} #{to_a}>"
+      end
+
+      def to_a
+        keys
+      end
+
+      def add_metric(metric_id)
+        with_standard_id(metric_id) do |cube_name, metric_name|
+          register cube.find_metric(metric_name) if cube_name == sub_cube.cube_name
+        end
+      end
+
+      def built_in_metrics
+        values.select{|x| x.is_a?(Schema::BuiltInMetric)}
+      end
+
+      def custom_metrics
+        values.select{|x| x.is_a?(Schema::CustomMetric)}
+      end
+
+      def custom_rollups
+        values.select{|x| x.is_a?(Schema::CustomRollup)}
+      end
+
+      def metric_ids
+        values.map(&:id)
+      end
+
+      def metric_objects
+        values
+      end
+
+    end
+  end
+end

data/lib/martyr/runtime/sub_cubes/sub_cube.rb

@@ -0,0 +1,134 @@
+module Martyr
+  module Runtime
+    class SubCube
+
+      include Martyr::LevelComparator
+      include Martyr::Translations
+      include Martyr::Runtime::FactFillerStrategies
+
+      attr_reader :query_context, :cube, :fact_scopes, :metrics, :grain
+      delegate :combined_sql, :test, to: :fact_scopes
+
+      # TODO: supported_* methods are delegated to the grain, but there are equivalent methods in the cube that mean
+      #       something else and sometimes needed. #select_supported_level_ids, for instance, is relying on those
+      #       methods in the cube, not the grain.
+      #       This is confusing.
+      delegate :cube_name, :dimension_associations, :select_supported_level_ids, :standardizer, to: :cube
+      delegate :supported_level_associations, :supported_level_definitions, :has_association_with_level?, to: :grain
+
+      delegate :find_metric, :metric_ids, :metric_objects, :built_in_metrics, :custom_metrics, to: :metrics
+      delegate :facts, to: :fact_indexer
+      delegate :definition_from_id, to: :query_context
+
+      alias_method :dimension_bus, :query_context
+
+      # @param cube [Martyr::Cube]
+      def initialize(query_context, cube)
+        @query_context = query_context
+        @cube = cube
+        @metrics = QueryMetrics.new(self)
+        @grain = SubCubeGrain.new(self)
+      end
+
+      def inspect
+        to_hash.inspect
+      end
+
+      def to_hash
+        {cube_name => {metrics: metrics.to_a, grain: grain.to_a}}
+      end
+
+      def dimension_definitions
+        cube.supported_dimension_definitions
+      end
+
+      # @return [DimensionReference, LevelAssociation]
+      def association_from_id(id)
+        with_standard_id(id) do |x, y|
+          return dimension_associations[x].try(:levels).try(:[], y) if x and y
+          dimension_associations[x]
+        end
+      end
+
+      def common_denominator_level_association(level_id, prefer_query: false)
+        @_common_denominator_level_association ||= {}
+        return @_common_denominator_level_association[level_id] if @_common_denominator_level_association[level_id]
+
+        level = definition_from_id(level_id)
+        dimension_association = dimension_associations.find_dimension_association(level.dimension_name)
+        level = find_common_denominator_level(level, dimension_association.level_objects, prefer_query: prefer_query)
+        return nil unless level
+
+        @_common_denominator_level_association[level_id] = dimension_association.levels[level.name]
+      end
+
+      # = Definitions
+
+      def set_metrics(metrics_arr)
+        return unless metrics_arr.present?
+        metrics_arr.each do |metric_id|
+          @metrics.add_metric(metric_id)
+        end
+      end
+
+      def set_grain(grain_arr)
+        select_supported_level_ids(grain_arr).each do |level_id|
+          @grain.add_granularity(level_id)
+        end
+      end
+
+      def set_sub_facts(sub_facts_arr)
+        @fact_scopes = cube.build_fact_scopes(sub_facts_arr)
+      end
+
+      def lowest_level_ids_in_grain
+        grain.level_ids
+      end
+
+      # @param data_slice [DataSlice] that is scoped to the cube
+      def decorate_all_scopes(data_slice)
+        grain.add_to_select(fact_scopes)
+        metrics.add_to_select(fact_scopes)
+        data_slice.add_to_where(fact_scopes, dimension_bus)
+        grain.add_to_group_by(fact_scopes)
+      end
+
+      # = Running
+
+      # @param memory_slice [MemorySlice]
+      # @option levels [Array<String, Martyr::Level>] array of level IDs or any type of level to group facts by.
+      #   Default is all levels in the query context.
+      # @option metrics [Array<String, BaseMetric>] array of metric IDs or metric objects to roll up in the elements.
+      def elements(memory_slice, levels: nil, metrics: nil)
+        element_locator_for(memory_slice, metrics: metrics).all(sanitize_levels(levels: levels).map(&:id))
+      end
+
+      def element_locator_for(memory_slice, metrics: nil)
+        ElementLocator.new memory_slice: memory_slice, metrics: sanitize_metrics(metrics: metrics),
+          fact_indexer: fact_indexer, helper_module: query_context.element_helper_module,
+          restrict_level_ids: grain.supported_level_definition_ids, standardizer: standardizer
+      end
+
+      def fact_indexer
+        @fact_indexer ||= FactIndexer.new(self, grain.null? ? [] : fact_scopes.run.map { |hash| Fact.new(self, hash) })
+      end
+
+      # @option metrics [Array<String, BaseMetric>] array of metric IDs or metric objects
+      def sanitize_metrics(metrics: nil)
+        metric_ids = Array.wrap(metrics).map { |x| to_id(x) }.presence || self.metrics.metric_ids
+        metric_ids = metric_ids & self.metrics.metric_ids
+        metric_ids.map {|id| query_context.metric(id) }
+      end
+
+      # @option levels [Array<String, Martyr::Level>] array of level IDs or any type of level to group facts by.
+      #   Default is all levels in grain
+      # @return [Array<String, BaseLevelScope>]
+      def sanitize_levels(levels: nil)
+        level_ids = levels.nil? ? query_context.level_ids_in_grain : Array.wrap(levels).map { |x| to_id(x) }
+        level_ids = select_supported_level_ids(level_ids)
+        query_context.levels_and_above_for(level_ids)
+      end
+
+    end
+  end
+end

data/lib/martyr/runtime/sub_cubes/sub_cube_grain.rb

@@ -0,0 +1,117 @@
+module Martyr
+  module Runtime
+    class SubCubeGrain
+      include Martyr::LevelComparator
+
+      attr_reader :sub_cube, :grain
+      delegate :cube, to: :sub_cube
+
+      def initialize(sub_cube)
+        @sub_cube = sub_cube
+        @grain = cube.default_fact_grain_level_associations.index_by{|x| x.dimension_name}
+        @null = false
+      end
+
+      def inspect
+        "#<#{self.class} #{to_a}>"
+      end
+
+      def to_a
+        supported_level_associations.map(&:id)
+      end
+
+      def level_ids
+        grain.values.map(&:id)
+      end
+
+      def null?
+        @null
+      end
+
+      # Maintains for every dimension the lowest supported level
+      def add_granularity(level_id)
+        level_to_add = sub_cube.common_denominator_level_association(level_id)
+        @null = true and return unless level_to_add
+        dimension = level_to_add.dimension_name
+        @grain[dimension] = more_detailed_level(@grain[dimension], level_to_add)
+      end
+
+      # TODO: remove
+      # def set_all_if_empty
+      #   return if @grain.present?
+      #   sub_cube.dimension_associations.each do |dimension_name, dimension_object|
+      #     @grain[dimension_name.to_s] = dimension_object.lowest_level
+      #   end
+      # end
+
+      # TODO: delete
+      # def nullify_scope_if_null(fact_scopes)
+      #   fact_scopes.set_null_scope if null?
+      # end
+
+      # Adds all supported levels including and above the sliced level
+      # @param fact_scopes [Runtime::FactScopeCollection]
+      def add_to_select(fact_scopes)
+        supported_level_associations.each do |level_object|
+          fact_scopes.add_select_operator_for_dimension do |operator|
+            operator.add_select(level_object.fact_key, as: level_object.fact_alias)
+          end
+        end
+      end
+
+      # @param fact_scopes [Runtime::FactScopeCollection]
+      def add_to_group_by(fact_scopes)
+        supported_level_associations.each do |level_object|
+          fact_scopes.add_group_operator do |operator|
+            operator.add_group(level_object.fact_alias)
+          end
+        end
+      end
+
+      # Assume the following levels, where (*) denotes has association to the fact
+      #   L1
+      #     L2 (*)
+      #       L3 (*)
+      #         L4 (*)
+      #
+      # Assuming the lowest level in the grain is L3:
+      #
+      #   supported_level_associations
+      #   # => [L2, L3] of LevelAssociation objects
+      #
+      #   supported_level_definitions
+      #   # => [L1, L2, L3] of BaseLevelDefinition objects
+      #
+      #   [L1, L2, L3, L4].map{|x| x.has_association_with_level(x) }
+      #   # => [false, true, true, false]
+      #
+
+      def supported_level_associations
+        return [] if null?
+        @supported_level_associations ||= grain.flat_map do |_dimension_name, lowest_level|
+          sub_cube.association_from_id(lowest_level.id).level_and_above
+        end
+      end
+
+      def supported_level_definitions
+        return [] if null?
+        @_supported_level_definitions ||= grain.flat_map do |_dimension_name, lowest_level|
+          sub_cube.definition_from_id(lowest_level.id).level_and_above
+        end
+      end
+
+      def supported_level_definition_ids
+        supported_level_definitions.map(&:id)
+      end
+
+      def supported_level_associations_lookup
+        @_supported_level_associations_lookup ||= supported_level_associations.index_by(&:id)
+      end
+
+      def has_association_with_level?(level_id)
+        !!supported_level_associations_lookup[level_id]
+      end
+
+    end
+  end
+end

data/lib/martyr/schema/dimension_associations/dimension_association_collection.rb

@@ -0,0 +1,33 @@
+module Martyr
+  module Schema
+    class DimensionAssociationCollection < HashWithIndifferentAccess
+      include Martyr::Registrable
+      include Martyr::Translations
+
+      attr_reader :dimension_definitions
+      alias_method :find_dimension_association, :find_or_error
+      alias_method :supports_dimension?, :has_key?
+
+      # @param dimension_definitions [DimensionDefinitionCollection]
+      def initialize(dimension_definitions)
+        @dimension_definitions = dimension_definitions
+      end
+
+      # @return [LevelAssociation]
+      def has_dimension_level(dimension_name, level_name, **args)
+        if has_key?(dimension_name)
+          dimension = find_or_nil(dimension_name)
+        else
+          dimension = dimension_definitions.find_dimension(dimension_name)
+          dimension = Martyr::DimensionReference.new(dimension, LevelAssociationCollection)
+          register dimension
+        end
+        dimension.has_dimension_level(level_name, **args)
+      end
+
+      def find_level_association(level_id)
+        with_standard_id(level_id) {|dimension, level| find_dimension_association(dimension).find_level(level)}
+      end
+    end
+  end
+end

data/lib/martyr/schema/dimension_associations/level_association.rb

@@ -0,0 +1,37 @@
+module Martyr
+  module Schema
+    class LevelAssociation
+      include Martyr::Level
+
+      attr_accessor :level, :fact_key, :fact_alias, :sort
+      alias_method :level_definition, :level
+
+      # Important so that the to_i will take into account all levels defined for the dimension, not just the supported one
+      delegate :to_i, to: :level
+
+      delegate :label_key, :label_expression, to: :level
+
+      # @param collection [LevelAssociationCollection]
+      # @param level [BaseLevelDefinition]
+      def initialize(collection, level, fact_key: nil, fact_alias: nil, sort: nil)
+        @collection = collection
+        @level = level
+        @fact_key = fact_key || level.fact_key
+        @fact_alias = fact_alias || level.fact_alias
+        @sort = sort || level.sort
+      end
+
+      def supported?
+        true
+      end
+
+      private
+
+      # Delegate everything to level
+      def method_missing(method_name, *args, &block)
+        level.respond_to?(method_name) ? level.send(method_name, *args, &block) : super
+      end
+
+    end
+  end
+end

data/lib/martyr/schema/dimension_associations/level_association_collection.rb

@@ -0,0 +1,18 @@
+module Martyr
+  module Schema
+    class LevelAssociationCollection < HashWithIndifferentAccess
+      include Martyr::LevelCollection
+
+      # @param level [String, Symbol]
+      # @return [LevelAssociation]
+      def has_dimension_level(level, **args)
+        level_definition = dimension_definition.levels[level]
+        raise Schema::Error.new("Could not find level `#{level}` for dimension #{dimension_name}") unless level_definition
+        level_association = LevelAssociation.new(self, level_definition, **args)
+        register level_association
+        arr = sort_by{|_name, level| level.to_i}
+        clear.merge!(Hash[arr])
+      end
+    end
+  end
+end

data/lib/martyr/schema/dimensions/dimension_definition_collection.rb

@@ -0,0 +1,39 @@
+module Martyr
+  module Schema
+    class DimensionDefinitionCollection < HashWithIndifferentAccess
+      include Martyr::Translations
+      include Martyr::Registrable
+
+      alias_method :supports_dimension?, :has_key?
+      alias_method :find_dimension, :find_or_error
+      alias_method :all, :to_hash
+
+      # @return [DimensionDefinition]
+      def define_dimension(*args, &block)
+        register PlainDimensionDefinition.new(*args, &block)
+      end
+
+      def find_level_definition(level_id)
+        with_standard_id(level_id) { |dimension, level| find_dimension(dimension).find_level(level) }
+      end
+
+      # # @param name [String, Symbol]
+      # # @return [DimensionDefinition] object that was found by traversing up the lookup tree
+      # def recursive_lookup(name)
+      #   fetch(name.to_s, parent_dimension_definitions.try(:recursive_lookup, name.to_s))
+      # end
+      #
+      # def find_dimension(name)
+      #   recursive_lookup(name) || raise(Schema::Error.new("Could not find dimension `#{name}`"))
+      # end
+      #
+
+      # # @return [Hash] { dimension_name => PlainDimensionDefinition } including dimension definitions from all superclasses
+      # def all
+      #   return to_hash unless parent_dimension_definitions
+      #   to_hash.merge(parent_dimension_definitions.all)
+      # end
+
+    end
+  end
+end

data/lib/martyr/schema/dimensions/plain_dimension_definition.rb

@@ -0,0 +1,39 @@
+module Martyr
+  module Schema
+    class PlainDimensionDefinition
+      include ActiveModel::Model
+      include Martyr::HasLevelCollection
+
+      attr_accessor :name, :title
+      delegate :degenerate_level, :query_level, to: :levels
+
+      # @param name [String]
+      # @option title [String]
+      def initialize(name, **options, &block)
+        super name: name.to_s,
+              title: options[:title] || name.to_s.titleize
+
+        @levels = LevelDefinitionCollection.new(dimension: self)
+        instance_eval(&block) if block
+      end
+
+      def dimension_definition
+        self
+      end
+
+      # For reflection
+      def dimension?
+        true
+      end
+
+      def build_data_slice(*args)
+        Runtime::PlainDimensionDataSlice.new(self, *args)
+      end
+
+      def build_memory_slice(*args)
+        Runtime::PlainDimensionMemorySlice.new(self, *args)
+      end
+
+    end
+  end
+end