eps 0.2.1 → 0.3.0

data/lib/eps/evaluators/lightgbm.rb

@@ -0,0 +1,116 @@
+module Eps
+  module Evaluators
+    class LightGBM
+      attr_reader :features
+
+      def initialize(trees:, objective:, labels:, features:, text_features:)
+        @trees = trees
+        @objective = objective
+        @labels = labels
+        @features = features
+        @text_features = text_features
+      end
+
+      def predict(data)
+        rows = data.map(&:to_h)
+
+        # sparse matrix
+        @text_features.each do |k, v|
+          encoder = TextEncoder.new(v)
+
+          values = data.columns.delete(k)
+          counts = encoder.transform(values)
+
+          encoder.vocabulary.each do |word|
+            data.columns[[k, word]] = [0] * values.size
+          end
+
+          counts.each_with_index do |xc, i|
+            row = rows[i]
+            row.delete(k)
+            xc.each do |word, count|
+              row[[k, word]] = count
+            end
+          end
+        end
+
+        case @objective
+        when "regression"
+          sum_trees(rows, @trees)
+        when "binary"
+          sum_trees(rows, @trees).map { |s| @labels[sigmoid(s) > 0.5 ? 1 : 0] }
+        else
+          tree_scores = []
+          num_trees = @trees.size / @labels.size
+          @trees.each_slice(num_trees).each do |trees|
+            tree_scores << sum_trees(rows, trees)
+          end
+          data.size.times.map do |i|
+            v = tree_scores.map { |s| s[i] }
+            idx = v.map.with_index.max_by { |v2, _| v2 }.last
+            @labels[idx]
+          end
+        end
+      end
+
+      private
+
+      def sum_trees(data, trees)
+        data.map do |row|
+          sum = 0
+          trees.each do |node|
+            score = node_score(node, row)
+            sum += score
+          end
+          sum
+        end
+      end
+
+      def matches?(node, row)
+        if node.predicate.nil?
+          true
+        else
+          v = row[node.field]
+
+          # sparse text feature
+          v = 0 if v.nil? && node.field.is_a?(Array)
+
+          if v.nil?
+            # missingValueStrategy="none"
+            false
+          else
+            case node.operator
+            when "equal"
+              v == node.value
+            when "in"
+              node.value.include?(v)
+            when "greaterThan"
+              v > node.value
+            when "lessOrEqual"
+              v <= node.value
+            else
+              raise "Unknown operator: #{node.operator}"
+            end
+          end
+        end
+      end
+
+      def node_score(node, row)
+        if matches?(node, row)
+          node.children.each do |c|
+            score = node_score(c, row)
+            return score if score
+          end
+          # noTrueChildStrategy="returnLastPrediction"
+          node.score
+        else
+          nil
+        end
+      end
+
+      def sigmoid(x)
+        1.0 / (1 + Math::E**(-x))
+      end
+    end
+  end
+end

data/lib/eps/evaluators/linear_regression.rb

@@ -0,0 +1,54 @@
+module Eps
+  module Evaluators
+    class LinearRegression
+      attr_reader :features
+
+      def initialize(coefficients:, features:, text_features:)
+        @coefficients = Hash[coefficients.map { |k, v| [k.is_a?(Array) ? [k[0].to_s, k[1]] : k.to_s, v] }]
+        @features = features
+        @text_features = text_features || {}
+      end
+
+      def predict(x)
+        intercept = @coefficients["_intercept"]
+        scores = [intercept] * x.size
+
+        @features.each do |k, type|
+          raise "Missing data in #{k}" if !x.columns[k] || x.columns[k].any?(&:nil?)
+
+          case type
+          when "categorical"
+            x.columns[k].each_with_index do |xv, i|
+              scores[i] += @coefficients[[k, xv]].to_f
+            end
+          when "text"
+            encoder = TextEncoder.new(@text_features[k])
+            counts = encoder.transform(x.columns[k])
+            coef = {}
+            @coefficients.each do |k2, v|
+              next unless k2.is_a?(Array) && k2.first == k
+              coef[k2.last] = v
+            end
+
+            counts.each_with_index do |xc, i|
+              xc.each do |word, count|
+                scores[i] += coef[word] * count if coef[word]
+              end
+            end
+          else
+            coef = @coefficients[k].to_f
+            x.columns[k].each_with_index do |xv, i|
+              scores[i] += coef * xv
+            end
+          end
+        end
+
+        scores
+      end
+
+      def coefficients
+        Hash[@coefficients.map { |k, v| [Array(k).join.to_sym, v] }]
+      end
+    end
+  end
+end

data/lib/eps/evaluators/naive_bayes.rb

@@ -0,0 +1,95 @@
+module Eps
+  module Evaluators
+    class NaiveBayes
+      attr_reader :features, :probabilities
+
+      def initialize(probabilities:, features:, derived: nil, legacy: false)
+        @probabilities = probabilities
+        @features = features
+        @derived = derived
+        @legacy = legacy
+      end
+
+      def predict(x)
+        probs = calculate_class_probabilities(x)
+        probs.map do |xp|
+          # convert probabilities
+          # not needed when just returning label
+          # sum = xp.values.map { |v| Math.exp(v) }.sum.to_f
+          # p xp.map { |k, v| [k, Math.exp(v) / sum] }.to_h
+          xp.sort_by { |k, v| [-v, k] }[0][0]
+        end
+      end
+
+      # use log to prevent underflow
+      # https://www.antoniomallia.it/lets-implement-a-gaussian-naive-bayes-classifier-in-python.html
+      def calculate_class_probabilities(x)
+        probs = Eps::DataFrame.new
+
+        # assign very small probability if probability is 0
+        tiny_p = @legacy ? 0.0001 : 0
+
+        total = probabilities[:prior].values.sum.to_f
+        probabilities[:prior].each do |c, cv|
+          prior = Math.log(cv / total)
+          px = [prior] * x.size
+
+          @features.each do |k, type|
+            case type
+            when "categorical"
+              x.columns[k].each_with_index do |xi, i|
+                vc = probabilities[:conditional][k][xi]
+
+                # unknown value if not vc
+                if vc
+                  denom = probabilities[:conditional][k].map { |k, v| v[c] }.sum.to_f
+                  p2 = vc[c].to_f / denom
+
+                  # TODO use proper smoothing instead
+                  p2 = tiny_p if p2 == 0
+
+                  px[i] += Math.log(p2)
+                end
+              end
+            when "derived"
+              @derived[k].each do |k2, v2|
+                vc = probabilities[:conditional][k2][c]
+
+                x.columns[k].each_with_index do |xi, i|
+                  px[i] += Math.log(calculate_probability(xi == v2 ? 1 : 0, vc[:mean], vc[:stdev]))
+                end
+              end
+            else
+              vc = probabilities[:conditional][k][c]
+
+              if vc[:stdev] != 0 && !vc[:stdev].nil?
+                x.columns[k].each_with_index do |xi, i|
+                  px[i] += Math.log(calculate_probability(xi, vc[:mean], vc[:stdev]))
+                end
+              else
+                x.columns[k].each_with_index do |xi, i|
+                  if xi != vc[:mean]
+                    # TODO use proper smoothing instead
+                    px[i] += Math.log(tiny_p)
+                  end
+                end
+              end
+            end
+
+            probs.columns[c] = px
+          end
+        end
+
+        probs
+      end
+
+      SQRT_2PI = Math.sqrt(2 * Math::PI)
+
+      # TODO memoize for performance
+      def calculate_probability(x, mean, stdev)
+        exponent = Math.exp(-((x - mean)**2) / (2 * (stdev**2)))
+        (1 / (SQRT_2PI * stdev)) * exponent
+      end
+    end
+  end
+end

data/lib/eps/evaluators/node.rb

@@ -0,0 +1,26 @@
+module Eps
+  module Evaluators
+    class Node
+      attr_accessor :score, :predicate, :children, :leaf_index
+
+      def initialize(predicate: nil, score: nil, children: nil, leaf_index: nil)
+        @predicate = predicate
+        @children = children || []
+        @score = score
+        @leaf_index = leaf_index
+      end
+
+      def field
+        @predicate[:field]
+      end
+
+      def operator
+        @predicate[:operator]
+      end
+
+      def value
+        @predicate[:value]
+      end
+    end
+  end
+end

data/lib/eps/label_encoder.rb

@@ -0,0 +1,41 @@
+module Eps
+  class LabelEncoder
+    attr_reader :labels
+
+    def initialize
+      @labels = {}
+    end
+
+    def fit(y)
+      labels = {}
+      y.compact.map(&:to_s).uniq.sort.each_with_index do |label, i|
+        labels[label] = i
+      end
+      @labels = labels
+    end
+
+    def fit_transform(y)
+      fit(y)
+      transform(y)
+    end
+
+    def transform(y)
+      y.map do |yi|
+        if yi.nil?
+          nil
+        else
+          v = @labels[yi.to_s]
+          raise "Unknown label: #{yi}" unless v
+          v
+        end
+      end
+    end
+
+    def inverse_transform(y)
+      inverse = Hash[@labels.map(&:reverse)]
+      y.map do |yi|
+        inverse[yi.to_i]
+      end
+    end
+  end
+end

data/lib/eps/lightgbm.rb

@@ -0,0 +1,237 @@
+require "eps/pmml_generators/lightgbm"
+
+module Eps
+  class LightGBM < BaseEstimator
+    include PmmlGenerators::LightGBM
+
+    def self.load_pmml(data)
+      super do |data|
+        objective = data.css("MiningModel").first.attribute("functionName").value
+        if objective == "classification"
+          labels = data.css("RegressionModel OutputField").map { |n| n.attribute("value").value }
+          objective = labels.size > 2 ? "multiclass" : "binary"
+        end
+
+        features = {}
+        text_features, derived_fields = extract_text_features(data, features)
+        node = data.css("DataDictionary").first
+        node.css("DataField")[1..-1].to_a.each do |node|
+          features[node.attribute("name").value] =
+            if node.attribute("optype").value == "categorical"
+              "categorical"
+            else
+              "numeric"
+            end
+        end
+
+        trees = []
+        data.css("Segmentation TreeModel").each do |tree|
+          node = find_nodes(tree.css("Node").first, derived_fields)
+          trees << node
+        end
+
+        Evaluators::LightGBM.new(trees: trees, objective: objective, labels: labels, features: features, text_features: text_features)
+      end
+    end
+
+    private
+
+    def _summary(extended: false)
+      str = String.new("")
+      importance = @booster.feature_importance
+      total = importance.sum.to_f
+      if total == 0
+        str << "Model needs more data for better predictions\n"
+      else
+        str << "Most important features\n"
+        @importance_keys.zip(importance).sort_by { |k, v| [-v, k] }.first(10).each do |k, v|
+          str << "#{display_field(k)}: #{(100 * v / total).round}\n"
+        end
+      end
+      str
+    end
+
+    def self.find_nodes(xml, derived_fields)
+      score = BigDecimal(xml.attribute("score").value).to_f
+
+      elements = xml.elements
+      xml_predicate = elements.first
+
+      predicate =
+        if xml_predicate.name == "True"
+          nil
+        elsif xml_predicate.name == "SimpleSetPredicate"
+          operator = "in"
+          value = xml_predicate.css("Array").text.scan(/"(.+?)(?<!\\)"|(\S+)/).flatten.compact.map { |v| v.gsub('\"', '"') }
+          field = xml_predicate.attribute("field").value
+          field = derived_fields[field] if derived_fields[field]
+          {
+            field: field,
+            operator: operator,
+            value: value
+          }
+        else
+          operator = xml_predicate.attribute("operator").value
+          value = xml_predicate.attribute("value").value
+          value = BigDecimal(value).to_f if operator == "greaterThan"
+          field = xml_predicate.attribute("field").value
+          field = derived_fields[field] if derived_fields[field]
+          {
+            field: field,
+            operator: operator,
+            value: value
+          }
+        end
+
+      children = elements[1..-1].map { |n| find_nodes(n, derived_fields) }
+
+      Evaluators::Node.new(score: score, predicate: predicate, children: children)
+    end
+
+    def _train(verbose: nil, early_stopping: nil)
+      train_set = @train_set
+      validation_set = @validation_set.dup
+      summary_label = train_set.label
+
+      # objective
+      objective =
+        if @target_type == "numeric"
+          "regression"
+        else
+          label_encoder = LabelEncoder.new
+          train_set.label = label_encoder.fit_transform(train_set.label)
+          validation_set.label = label_encoder.transform(validation_set.label) if validation_set
+          labels = label_encoder.labels.keys
+
+          if labels.size > 2
+            "multiclass"
+          else
+            "binary"
+          end
+        end
+
+      # label encoding
+      label_encoders = {}
+      @features.each do |k, type|
+        if type == "categorical"
+          label_encoder = LabelEncoder.new
+          train_set.columns[k] = label_encoder.fit_transform(train_set.columns[k])
+          validation_set.columns[k] = label_encoder.transform(validation_set.columns[k]) if validation_set
+          label_encoders[k] = label_encoder
+        end
+      end
+
+      # text feature encoding
+      prep_text_features(train_set)
+      prep_text_features(validation_set) if validation_set
+
+      # create params
+      params = {objective: objective}
+      params[:num_classes] = labels.size if objective == "multiclass"
+      if train_set.size < 30
+        params[:min_data_in_bin] = 1
+        params[:min_data_in_leaf] = 1
+      end
+
+      # create datasets
+      categorical_idx = @features.values.map.with_index.select { |type, _| type == "categorical" }.map(&:last)
+      train_ds = ::LightGBM::Dataset.new(train_set.map_rows(&:to_a), label: train_set.label, categorical_feature: categorical_idx, params: params)
+      validation_ds = ::LightGBM::Dataset.new(validation_set.map_rows(&:to_a), label: validation_set.label, categorical_feature: categorical_idx, params: params, reference: train_ds) if validation_set
+
+      # train
+      valid_sets = [train_ds]
+      valid_sets << validation_ds if validation_ds
+      valid_names = ["training"]
+      valid_names << "validation" if validation_ds
+      early_stopping = 50 if early_stopping.nil? && validation_ds
+      early_stopping = nil if early_stopping == false
+      booster = ::LightGBM.train(params, train_ds, num_boost_round: 1000, early_stopping_rounds: early_stopping, valid_sets: valid_sets, valid_names: valid_names, verbose_eval: verbose || false)
+
+      # separate summary from verbose_eval
+      puts if verbose
+
+      @importance_keys = train_set.columns.keys
+
+      # create evaluator
+      @label_encoders = label_encoders
+      booster_tree = JSON.parse(booster.to_json)
+      trees = booster_tree["tree_info"].map { |s| parse_tree(s["tree_structure"]) }
+      # reshape
+      if objective == "multiclass"
+        new_trees = []
+        grouped = trees.each_slice(labels.size).to_a
+        labels.size.times do |i|
+          new_trees.concat grouped.map { |v| v[i] }
+        end
+        trees = new_trees
+      end
+
+      # for pmml
+      @objective = objective
+      @labels = labels
+      @feature_importance = booster.feature_importance
+      @trees = trees
+      @booster = booster
+
+      # reset pmml
+      @pmml = nil
+
+      Evaluators::LightGBM.new(trees: trees, objective: objective, labels: labels, features: @features, text_features: @text_features)
+    end
+
+    def evaluator_class
+      PmmlLoaders::LightGBM
+    end
+
+    # for evaluator
+
+    def parse_tree(node)
+      if node["leaf_value"]
+        score = node["leaf_value"]
+        Evaluators::Node.new(score: score, leaf_index: node["leaf_index"])
+      else
+        field = @importance_keys[node["split_feature"]]
+        operator =
+          case node["decision_type"]
+          when "=="
+            "equal"
+          when "<="
+            node["default_left"] ? "greaterThan" : "lessOrEqual"
+          else
+            raise "Unknown decision type: #{node["decision_type"]}"
+          end
+
+        value =
+          if operator == "equal"
+            if node["threshold"].include?("||")
+              operator = "in"
+              @label_encoders[field].inverse_transform(node["threshold"].split("||"))
+            else
+              @label_encoders[field].inverse_transform([node["threshold"]])[0]
+            end
+          else
+            node["threshold"]
+          end
+
+        predicate = {
+          field: field,
+          value: value,
+          operator: operator
+        }
+
+        left = parse_tree(node["left_child"])
+        right = parse_tree(node["right_child"])
+
+        if node["default_left"]
+          right.predicate = predicate
+          left.children.unshift right
+          left
+        else
+          left.predicate = predicate
+          right.children.unshift left
+          right
+        end
+      end
+    end
+  end
+end