martyr 0.1.74.pre

data/lib/martyr/schema/dimensions/time_dimension_definition.rb

@@ -0,0 +1,24 @@
+module Martyr
+  module Schema
+    class TimeDimension
+
+      # @param name [Symbol, String]
+      # @param column [Symbol, String]
+      def initialize(name, column: name)
+        # TODO: convert the column into level definitions of week, month, and year
+        # super(name: name.to_s) do |dimension|
+        #   dimension.add_level column, foreign_key: column
+        # end
+      end
+
+      def build_data_slice(*args)
+        Runtime::TimeDimensionDataSlice.new self, *args
+      end
+
+      def find_level(name)
+        # TODO: implement
+      end
+
+    end
+  end
+end

data/lib/martyr/schema/facts/base_fact_definition.rb

@@ -0,0 +1,22 @@
+module Martyr
+  module Schema
+    class BaseFactDefinition
+      attr_reader :cube, :scope, :dimension_associations, :joins_by_default
+      alias_method :dimensions, :dimension_associations
+
+      def supports_dimension_level?(dimension_name, level_name)
+        dimension = dimension_associations[dimension_name]
+        return false unless dimension
+
+        lowest_supported_level_i = dimension.lowest_level.to_i
+        considered_level_i = dimension_definitions.find_dimension(dimension_name).find_level(level_name).to_i
+        considered_level_i <= lowest_supported_level_i
+      end
+
+      def has_dimension_level?(dimension_name, level_name)
+        dimension = dimension_associations[dimension_name]
+        dimension and dimension.has_level?(level_name)
+      end
+    end
+  end
+end

data/lib/martyr/schema/facts/fact_definition_collection.rb

@@ -0,0 +1,44 @@
+module Martyr
+  module Schema
+    class FactDefinitionCollection < HashWithIndifferentAccess
+      include Martyr::Registrable
+
+      attr_reader :cube
+
+      def initialize(cube)
+        @cube = cube
+      end
+
+      def main_fact
+        fetch(:main, nil) || build_main_fact
+      end
+
+      def build_main_fact
+        register MainFactDefinition.new(cube)
+      end
+
+      def sub_fact(name, &block)
+        raise Schema::Error.new('`main` is a reserved query name') if name.to_s == 'main'
+        register SubFactDefinition.new(cube, name, &block)
+      end
+      alias_method :sub_query, :sub_fact
+
+      # @param selected_sub_facts [Array<String>] array of sub-fact keys to include in the returned collection.
+      # @return [Runtime::FactScopeCollection]
+      def build_fact_scopes(selected_sub_facts = [])
+        selected_sub_facts_hash = Hash[selected_sub_facts.map{|x| [x.to_s, true]}]
+        missing_sub_facts = selected_sub_facts_hash.keys - self.keys
+
+        raise Schema::Error.new("Could not find #{'sub query'.pluralize(missing_sub_facts.length)} " +
+          missing_sub_facts.join(', ')) if missing_sub_facts.present?
+
+        scope_collection = Runtime::FactScopeCollection.new
+        values.each do |scope_definition|
+          next unless scope_definition.joins_by_default or selected_sub_facts_hash[scope_definition.name]
+          scope_collection.register scope_definition.build
+        end
+        scope_collection
+      end
+    end
+  end
+end

data/lib/martyr/schema/facts/main_fact_definition.rb

@@ -0,0 +1,45 @@
+module Martyr
+  module Schema
+    class MainFactDefinition < BaseFactDefinition
+
+      delegate :dimension_definitions, to: :@cube
+
+      delegate :supports_metric?, to: :metric_definitions
+      delegate :supports_dimension?, to: :dimension_associations
+
+      # = DSL
+
+      delegate :has_dimension_level, :find_dimension_association, :find_level_association, to: :dimension_associations
+      delegate :find_metric, :has_count_distinct_metric, :has_min_metric, :has_max_metric, :has_sum_metric,
+               :has_custom_metric, :has_custom_rollup, to: :metric_definitions
+
+      # @param cube [Martyr::Cube]
+      def initialize(cube)
+        @cube = cube
+        @joins_by_default = true
+      end
+
+      def dimension_associations
+        @dimension_associations ||= DimensionAssociationCollection.new(dimension_definitions)
+      end
+
+      def metric_definitions
+        @metric_definitions ||= Schema::MetricDefinitionCollection.new(@cube)
+      end
+      alias_method :metrics, :metric_definitions
+
+      def main_query(&scope)
+        @scope = scope
+      end
+
+      def name
+        'main'
+      end
+
+      # @return [Runtime::MainFactScope]
+      def build
+        Runtime::MainFactScope.new(self)
+      end
+    end
+  end
+end

data/lib/martyr/schema/facts/sub_fact_definition.rb

@@ -0,0 +1,44 @@
+module Martyr
+  module Schema
+    class SubFactDefinition < BaseFactDefinition
+      attr_reader :name, :join_clause, :join_on
+
+      delegate :main_fact, :dimension_definitions, to: :cube
+
+      delegate :find_level_association, to: :dimension_associations
+
+      def initialize(cube, name, &block)
+        @cube = cube
+        @name = name.to_s
+        @dimension_associations = DimensionAssociationCollection.new(dimension_definitions)
+        @joins_by_default = false
+
+        scope = instance_eval(&block)
+        @scope = -> { scope }
+      end
+
+      def supports_metric?(*)
+        false
+      end
+
+      def has_dimension_level(dimension_name, level_name, **args)
+        raise Schema::Error.new("Dimension level `#{dimension_name}.#{level_name}` does not exist in main query") unless
+            main_fact.has_dimension_level?(dimension_name, level_name)
+
+        dimension_associations.has_dimension_level(dimension_name, level_name, **args)
+      end
+
+      def joins_with(join_clause, on:, by_default: false)
+        @join_clause = join_clause
+        @join_on = on
+        @joins_by_default = by_default
+      end
+
+      # @return [Runtime::SubFactScope]
+      def build
+        Runtime::SubFactScope.new(self)
+      end
+
+    end
+  end
+end

data/lib/martyr/schema/metrics/base_metric.rb

@@ -0,0 +1,77 @@
+module Martyr
+  module Schema
+    class BaseMetric
+      include ActiveModel::Model
+      extend Martyr::Translations
+
+      attr_accessor :cube_name, :name, :rollup_function, :sort, :fact_grain
+
+      def id
+        "#{cube_name}.#{name}"
+      end
+
+      alias_method :slice_id, :id
+
+      # Used for reflection
+      def metric?
+        true
+      end
+
+      def human_name
+        name.to_s.titleize
+      end
+
+      def build_data_slice(*)
+        raise NotImplementedError
+      end
+
+      def build_memory_slice(*)
+        raise NotImplementedError
+      end
+
+      # @param fact_scopes [Runtime::FactScopeCollection]
+      def add_to_select(fact_scopes)
+        raise NotImplementedError
+      end
+
+      def extract(fact)
+        raise NotImplementedError
+      end
+
+      # @param element [Runtime::Element]
+      def rollup(element)
+        case rollup_function.to_s
+          when 'count'
+            element.facts.length
+          when 'sum'
+            element.facts.map{|x| x.fetch(id) || 0}.reduce(:+)
+          when 'min'
+            element.facts.map{|x| x.fetch(id)}.compact.min
+          when 'max'
+            element.facts.map{|x| x.fetch(id)}.compact.max
+          when 'none'
+
+            # Two scenarios:
+            #   (1) The user does not specify a fact_grain on a rollup: :none custom metric
+            #         The custom metric operates on the fact grain and is not rolled up when the element contains
+            #         multiple facts. We return it only if the facts length is 1.
+            #
+            #   (2) fact_grain exists
+            #         The custom metric operates on the levels provided by the user. There are three scenarios depending
+            #         on the element grain:
+            #         - Same level as the fact grain
+            #             There is only one fact, and facts.first returns it
+            #
+            #         - More detailed level than the fact grain
+            #             We should be able to pick any random fact.
+            #
+            #         - Less detailed than the fact grain
+            #             If there is only one fact, we return its value, otherwise we avoid rolling up and return nil
+            #
+            element.facts.first.fetch(id) if element.facts.length == 1 or
+              (fact_grain.present? and (fact_grain - element.grain_level_ids).empty?)
+        end
+      end
+    end
+  end
+end

data/lib/martyr/schema/metrics/built_in_metric.rb

@@ -0,0 +1,38 @@
+module Martyr
+  module Schema
+    class BuiltInMetric < BaseMetric
+
+      attr_accessor :statement, :fact_alias, :typecast, :sub_queries
+
+      def build_data_slice(*args)
+        Runtime::MetricDataSlice.new(self, *args)
+      end
+
+      def build_memory_slice(*args)
+        Runtime::MetricMemorySlice.new(self, *args)
+      end
+
+      # @param fact_scopes [Runtime::FactScopeCollection]
+      def add_to_select(fact_scopes)
+        fact_scopes.add_select_operator_for_metric(name) do |operator|
+          operator.add_select(statement, as: fact_alias, data_rollup_sql: data_rollup_sql)
+        end
+      end
+
+      def extract(fact)
+        fact.raw.fetch(fact_alias.to_s).try(:send, typecast || :to_i)
+      end
+
+      private
+
+      def data_rollup_sql
+        if rollup_function.to_s == 'none'
+          fact_alias
+        else
+          "#{rollup_function.to_s.upcase}(#{fact_alias}) AS #{fact_alias}"
+        end
+      end
+
+    end
+  end
+end

data/lib/martyr/schema/metrics/count_distinct_metric.rb

@@ -0,0 +1,172 @@
+module Martyr
+  module Schema
+    class CountDistinctMetric < BuiltInMetric
+
+      # Allows predetermining and caching the rollup strategy that will be used to rollup metrics of elements. This has
+      # performance benefits when rolling up a large array of elements that were the result of the same query.
+      #
+      # Usage:
+      #
+      #   CountDistinctMetric.enable_rollup_strategy_caching(metrics) do
+      #     # Code use to rollup (compute) metrics of many elements
+      #   end
+      #
+      # @param metrics [Array<BaseMetric>] any metric that is planned to be rolled up. The subset of count distinct
+      #   metrics will be used.
+      # @param element_grain [Array<String>] array of level IDs
+      # @param fact_grain [Array<String>] array of level IDs
+      def self.enable_rollup_strategy_caching(metrics)
+        relevant_metrics = metrics.select {|metric| metric.is_a?(self)}
+
+        relevant_metrics.each do |count_distinct_metric|
+          count_distinct_metric.send(:enable_rollup_strategy_caching)
+        end
+
+        yield
+
+      ensure
+        relevant_metrics.each do |count_distinct_metric|
+          count_distinct_metric.send(:cleanup_rollup_strategy)
+        end
+      end
+
+      # @attribute level [Schema::LevelAssociation]
+      # @attribute null_unless [String] @see MetricDefinitionCollection#has_count_distinct_metric
+      attr_accessor :level, :null_unless
+      delegate :id, to: :level, prefix: true
+
+      # @override
+      def add_to_select(fact_scopes)
+        # Adding an inner SQL statement with the metric's fact_alias is essential for metric slices
+        # Note that it is not used for the rollup in the wrapper
+        fact_scopes.add_select_operator_for_metric(name) do |operator|
+          operator.add_select(inner_sql_statement, as: fact_alias)
+        end
+
+        fact_scopes.add_select_operator_for_metric(name) do |operator|
+          operator.add_select(select_statement, as: inner_sql_helper_field, data_rollup_sql: data_rollup_sql)
+        end
+      end
+
+      # @override
+      def data_rollup_sql
+        "COUNT(DISTINCT #{inner_sql_helper_field}) AS #{fact_alias}"
+      end
+
+      # @override
+      def rollup(element)
+        rollup_strategy(element).run(element)
+      end
+
+      private
+
+      def inner_sql_statement
+        return '1' unless null_unless.present?
+        "CASE WHEN #{null_unless} THEN 1 ELSE 0 END"
+      end
+
+      def select_statement
+        return level.fact_key unless null_unless.present?
+        "CASE WHEN #{null_unless} THEN #{level.fact_key} ELSE NULL END"
+      end
+
+      def inner_sql_helper_field
+        return level.fact_alias unless null_unless.present?
+        [fact_alias, 'distinct_helper'].join('_')
+      end
+
+      # @return [RollupStrategy] cached version if caching is enabled, otherwise a newly minted object
+      def rollup_strategy(element)
+        return @rollup_strategy if @rollup_strategy_caching_enabled and @rollup_strategy
+        rollup_strategy = RollupStrategy.new(self, element.grain_level_ids, element.facts.first.grain_level_ids)
+        @rollup_strategy = rollup_strategy if @rollup_strategy_caching_enabled
+        rollup_strategy
+      end
+
+      def enable_rollup_strategy_caching
+        @rollup_strategy_caching_enabled = true
+      end
+
+      def cleanup_rollup_strategy
+        @rollup_strategy = nil
+        @rollup_strategy_caching_enabled = false
+      end
+
+      class RollupStrategy
+        attr_reader :metric, :strategy_method
+
+        # This is the trickiest part.
+        # Consider count distinct on `customers.last_name`.
+        #
+        # The following scenarios exist:
+        #   1. The level was not part of the fact to begin with.
+        #     There are some cases in which double counting cannot be avoided, hence rollup is not always possible.
+        #     For example, let's say we had a query on `media_types.name` and `genres.name`.
+        #     For every given combination, the fact holds a distinct count of the number of customers.
+        #     However, the same customer may have bought different combinations of media types and genres.
+        #     Therefore, if we remove `genres.name` from the element grain we have no way to correctly rollup the number
+        #     of distinct customers per media type.
+        #
+        #     There are two exceptions in which rollup is allowed:
+        #     (a) No levels were dropped between fact grain and element grain.
+        #     (b) Some levels were dropped, but all of them represent levels above `customers.last_name`.
+        #
+        #     If we had a query on `media_types.name` and `customers.country`, if we drop `customers.country` we can
+        #     perform a SUM of the facts.
+        #
+        #   2. The level was part of the fact and is also part of the element grain
+        #     Can be retrieved from the first fact. All facts should have the same value.
+        #
+        #   3. The level was part of the fact but is not part of the element grain
+        #     A DISTINCT count can be done in memory
+        #
+        def initialize(metric, element_grain, fact_grain)
+          @metric = metric
+
+          # Scenario 1
+          unless fact_grain.include?(metric.level_id)
+            acceptable_removed_level_ids = metric.level.level_and_above.map(&:id)
+            if (element_grain - fact_grain).all? { |level_id| acceptable_removed_level_ids.include?(level_id) }
+              @strategy_method = :sum
+            else
+              @strategy_method = :error
+            end
+            return
+          end
+
+          # Scenario 2
+          if element_grain.include?(metric.level_id)
+            @strategy_method = :first
+            return
+          end
+
+          # Scenario 3
+          @strategy_method = :uniq
+        end
+
+        def run(element)
+          send(strategy_method, element)
+        end
+
+        private
+
+        def sum(element)
+          element.facts.map{|fact| fact.fetch(metric.id)}.reduce(:+)
+        end
+
+        def error(_element)
+          "Invalid count distinct rollup. Add `#{metric.level_id}` to the grain"
+        end
+
+        def first(element)
+          element.facts.first.fetch(metric.id)
+        end
+
+        def uniq(element)
+          # We check for fetch(metric.id) > 0 for the sake of the use of +null_unless+ option
+          element.facts.select { |x| x.fetch(metric.id) > 0 }.map{ |x| x.fact_key_for(metric.level_id) }.uniq.length
+        end
+      end
+    end
+  end
+end

data/lib/martyr/schema/metrics/custom_metric.rb

@@ -0,0 +1,26 @@
+module Martyr
+  module Schema
+    class CustomMetric < BaseMetric
+
+      attr_accessor :block, :depends_on
+
+      def build_data_slice(*)
+        raise Runtime::Error.new("Custom metrics cannot be sliced: attempted on metric `#{name}`")
+      end
+
+      def build_memory_slice(*args)
+        Runtime::MetricMemorySlice.new(self, *args)
+      end
+
+      # @param fact_scopes [Runtime::FactScopeCollection]
+      def add_to_select(fact_scopes)
+        # no-op
+      end
+
+      def extract(fact)
+        fact.instance_exec(&block)
+      end
+
+    end
+  end
+end

data/lib/martyr/schema/metrics/custom_rollup.rb

@@ -0,0 +1,31 @@
+module Martyr
+  module Schema
+    class CustomRollup < BaseMetric
+      attr_accessor :cube_name, :name, :block, :depends_on
+
+      def build_data_slice(*)
+        raise Runtime::Error.new("Rollups cannot be sliced: attempted on rollup `#{name}`")
+      end
+
+      def build_memory_slice(*)
+        raise Runtime::Error.new("Rollups cannot be sliced: attempted on rollup `#{name}`")
+      end
+
+      # @param fact_scopes [Runtime::FactScopeCollection]
+      def add_to_select(fact_scopes)
+        # no-op
+      end
+
+      def extract(fact)
+        raise Runtime::Error.new("Rollups cannot be extracted: attempted on rollup `#{name}`")
+      end
+
+      # @override
+      # @param element [Runtime::Element]
+      def rollup(element)
+        element.instance_exec(&block)
+      end
+
+    end
+  end
+end

data/lib/martyr/schema/metrics/dependency_inferrer.rb

@@ -0,0 +1,150 @@
+module Martyr
+  module Schema
+    class DependencyInferrer
+
+      attr_reader :level_keys_hash, :metric_keys_hash
+
+      def initialize
+        @cubes = []
+        @level_keys_hash = {}
+        @metric_keys_hash = {}
+      end
+
+      def add_cube_levels(cube)
+        cube.supported_level_definitions.each do |level|
+          add_level(level)
+        end
+        self
+      end
+
+      # We rely on adding metric one-by-one to avoid infinite recursion
+      # @param metric [BaseMetric]
+      def add_metric(metric)
+        @metric_keys_hash[metric.name.to_s] = metric.id
+        @metric_keys_hash[metric.id.to_s] = metric.id
+      end
+
+      # @return [#depends_on, #fact_grain] an object responding to these methods
+      # The idea is that the block won't be evaluated if the user intervened with one of the options
+      def infer_from_block(depends_on: nil, fact_grain: nil, &block)
+        if (depends_on.nil? or depends_on == []) and (fact_grain.nil? or fact_grain == [])
+          evaluator = BlockEvaluator.new(self)
+          evaluator.instance_exec(&block)
+          evaluator
+        else
+          UserValues.new(depends_on, fact_grain)
+        end
+      end
+
+      private
+
+      # @param level [BaseLevelDefinition]
+      def add_level(level)
+        @level_keys_hash[level.id] = level.id
+        level.helper_methods.each do |method_name|
+          @level_keys_hash[method_name] = level.id
+        end
+      end
+
+      class UserValues
+        attr_reader :depends_on, :fact_grain
+
+        def initialize(depends_on, fact_grain)
+          @depends_on = depends_on == false ? [] : Array.wrap(depends_on)
+          @fact_grain = fact_grain == false ? [] : Array.wrap(fact_grain)
+        end
+      end
+
+      class BlockEvaluator
+        include Comparable
+
+        def initialize(inferrer)
+          @inferrer = inferrer
+          @depends_on_hash = {}
+          @fact_grain_hash = {}
+        end
+
+        def depends_on
+          @depends_on_hash.keys
+        end
+
+        def fact_grain
+          @fact_grain_hash.keys
+        end
+
+        def locate(*args)
+          main_arg = args.first
+          if main_arg.is_a?(String)
+            infer_either(main_arg)
+          elsif main_arg.is_a?(Hash)
+            main_arg.keys.each do |arg|
+              infer_either(arg)
+            end
+          end
+          self
+        end
+
+        def fetch(id)
+          infer_either(id)
+          self
+        end
+
+        def [](key)
+          infer_depends_on(key)
+          self
+        end
+
+        def key_for(level_id)
+          infer_fact_grain(level_id)
+          self
+        end
+
+        def record_for(level_id)
+          infer_fact_grain(level_id)
+          self
+        end
+
+        def fact_key_for(level_id)
+          infer_fact_grain(level_id)
+          self
+        end
+
+        def <=>(_other)
+          0
+        end
+
+        def coerce(other)
+          [other, other]
+        end
+
+        private
+
+        # @param candidate [String] a suspicious level ID that should be check against the white list
+        # @return [nil, true] true if added
+        def infer_fact_grain(candidate)
+          level_id = @inferrer.level_keys_hash[candidate.to_s]
+          return unless level_id.present?
+          @fact_grain_hash[level_id] = true
+        end
+
+        # @param candidate [String] a suspicious metric ID that should be check against the white list
+        # @return [nil, true] true if added
+        def infer_depends_on(candidate)
+          metric_id = @inferrer.metric_keys_hash[candidate.to_s]
+          return unless metric_id.present?
+          @depends_on_hash[metric_id] = true
+        end
+
+        def infer_either(candidate)
+          infer_fact_grain(candidate) || infer_depends_on(candidate)
+        end
+
+        def method_missing(name, *args, &block)
+          infer_either(name)
+          self
+        end
+      end
+
+    end
+  end
+end