eps 0.3.0 → 0.3.5

data/lib/eps/pmml/loader.rb

@@ -0,0 +1,241 @@
+module Eps
+  module PMML
+    class Loader
+      attr_reader :data
+
+      def initialize(pmml)
+        if pmml.is_a?(String)
+          pmml = Nokogiri::XML(pmml) { |config| config.strict }
+        end
+        @data = pmml
+      end
+
+      def load
+        if data.css("Segmentation").any?
+          lightgbm
+        elsif data.css("RegressionModel").any?
+          linear_regression
+        elsif data.css("NaiveBayesModel").any?
+          naive_bayes
+        else
+          raise "Unknown model"
+        end
+      end
+
+      private
+
+      def lightgbm
+        objective = data.css("MiningModel").first.attribute("functionName").value
+        if objective == "classification"
+          labels = data.css("RegressionModel OutputField").map { |n| n.attribute("value").value }
+          objective = labels.size > 2 ? "multiclass" : "binary"
+        end
+
+        features = {}
+        text_features, derived_fields = extract_text_features(data, features)
+        node = data.css("DataDictionary").first
+        node.css("DataField")[1..-1].to_a.each do |node|
+          features[node.attribute("name").value] =
+            if node.attribute("optype").value == "categorical"
+              "categorical"
+            else
+              "numeric"
+            end
+        end
+
+        trees = []
+        data.css("Segmentation TreeModel").each do |tree|
+          node = find_nodes(tree.css("Node").first, derived_fields)
+          trees << node
+        end
+
+        Evaluators::LightGBM.new(trees: trees, objective: objective, labels: labels, features: features, text_features: text_features)
+      end
+
+      def linear_regression
+        node = data.css("RegressionTable")
+
+        coefficients = {
+          "_intercept" => node.attribute("intercept").value.to_f
+        }
+
+        features = {}
+
+        text_features, derived_fields = extract_text_features(data, features)
+
+        node.css("NumericPredictor").each do |n|
+          name = n.attribute("name").value
+          if derived_fields[name]
+            name = derived_fields[name]
+          else
+            features[name] = "numeric"
+          end
+          coefficients[name] = n.attribute("coefficient").value.to_f
+        end
+
+        node.css("CategoricalPredictor").each do |n|
+          name = n.attribute("name").value
+          coefficients[[name, n.attribute("value").value]] = n.attribute("coefficient").value.to_f
+          features[name] = "categorical"
+        end
+
+        Evaluators::LinearRegression.new(coefficients: coefficients, features: features, text_features: text_features)
+      end
+
+      def naive_bayes
+        node = data.css("NaiveBayesModel")
+
+        prior = {}
+        node.css("BayesOutput TargetValueCount").each do |n|
+          prior[n.attribute("value").value] = n.attribute("count").value.to_f
+        end
+
+        legacy = false
+
+        conditional = {}
+        features = {}
+        node.css("BayesInput").each do |n|
+          prob = {}
+
+          # numeric
+          n.css("TargetValueStat").each do |n2|
+            n3 = n2.css("GaussianDistribution")
+            prob[n2.attribute("value").value] = {
+              mean: n3.attribute("mean").value.to_f,
+              stdev: Math.sqrt(n3.attribute("variance").value.to_f)
+            }
+          end
+
+          # detect bad form in Eps < 0.3
+          bad_format = n.css("PairCounts").map { |n2| n2.attribute("value").value } == prior.keys
+
+          # categorical
+          n.css("PairCounts").each do |n2|
+            if bad_format
+              n2.css("TargetValueCount").each do |n3|
+                prob[n3.attribute("value").value] ||= {}
+                prob[n3.attribute("value").value][n2.attribute("value").value] = n3.attribute("count").value.to_f
+              end
+            else
+              boom = {}
+              n2.css("TargetValueCount").each do |n3|
+                boom[n3.attribute("value").value] = n3.attribute("count").value.to_f
+              end
+              prob[n2.attribute("value").value] = boom
+            end
+          end
+
+          if bad_format
+            legacy = true
+            prob.each do |k, v|
+              prior.keys.each do |k|
+                v[k] ||= 0.0
+              end
+            end
+          end
+
+          name = n.attribute("fieldName").value
+          conditional[name] = prob
+          features[name] = n.css("TargetValueStat").any? ? "numeric" : "categorical"
+        end
+
+        target = node.css("BayesOutput").attribute("fieldName").value
+
+        probabilities = {
+          prior: prior,
+          conditional: conditional
+        }
+
+        # get derived fields
+        derived = {}
+        data.css("DerivedField").each do |n|
+          name = n.attribute("name").value
+          field = n.css("NormDiscrete").attribute("field").value
+          value = n.css("NormDiscrete").attribute("value").value
+          features.delete(name)
+          features[field] = "derived"
+          derived[field] ||= {}
+          derived[field][name] = value
+        end
+
+        Evaluators::NaiveBayes.new(probabilities: probabilities, features: features, derived: derived, legacy: legacy)
+      end
+
+      def extract_text_features(data, features)
+        # updates features object
+        vocabulary = {}
+        function_mapping = {}
+        derived_fields = {}
+        data.css("LocalTransformations DerivedField, TransformationDictionary DerivedField").each do |n|
+          name = n.attribute("name")&.value
+          field = n.css("FieldRef").attribute("field").value
+          value = n.css("Constant").text
+
+          field = field[10..-2] if field =~ /\Alowercase\(.+\)\z/
+          next if value.empty?
+
+          (vocabulary[field] ||= []) << value
+
+          function_mapping[field] = n.css("Apply").attribute("function").value
+
+          derived_fields[name] = [field, value]
+        end
+
+        functions = {}
+        data.css("TransformationDictionary DefineFunction").each do |n|
+          name = n.attribute("name").value
+          text_index = n.css("TextIndex")
+          functions[name] = {
+            tokenizer: Regexp.new(text_index.attribute("wordSeparatorCharacterRE").value),
+            case_sensitive: text_index.attribute("isCaseSensitive")&.value == "true"
+          }
+        end
+
+        text_features = {}
+        function_mapping.each do |field, function|
+          text_features[field] = functions[function].merge(vocabulary: vocabulary[field])
+          features[field] = "text"
+        end
+
+        [text_features, derived_fields]
+      end
+
+      def find_nodes(xml, derived_fields)
+        score = xml.attribute("score").value.to_f
+
+        elements = xml.elements
+        xml_predicate = elements.first
+
+        predicate =
+          if xml_predicate.name == "True"
+            nil
+          elsif xml_predicate.name == "SimpleSetPredicate"
+            operator = "in"
+            value = xml_predicate.css("Array").text.scan(/"(.+?)(?<!\\)"|(\S+)/).flatten.compact.map { |v| v.gsub('\"', '"') }
+            field = xml_predicate.attribute("field").value
+            field = derived_fields[field] if derived_fields[field]
+            {
+              field: field,
+              operator: operator,
+              value: value
+            }
+          else
+            operator = xml_predicate.attribute("operator").value
+            value = xml_predicate.attribute("value").value
+            value = value.to_f if operator == "greaterThan"
+            field = xml_predicate.attribute("field").value
+            field = derived_fields[field] if derived_fields[field]
+            {
+              field: field,
+              operator: operator,
+              value: value
+            }
+          end
+
+        children = elements[1..-1].map { |n| find_nodes(n, derived_fields) }
+
+        Evaluators::Node.new(score: score, predicate: predicate, children: children)
+      end
+    end
+  end
+end

data/lib/eps/version.rb

@@ -1,3 +1,3 @@
 module Eps
-  VERSION = "0.3.0"
+  VERSION = "0.3.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eps
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.5
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-05 00:00:00.000000000 Z
+date: 2020-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: lightgbm
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.5
+        version: 0.1.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.1.5
+        version: 0.1.7
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: numo-narray
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +108,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rover-df
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email: andrew@chartkick.com
 executables: []
@@ -117,7 +145,9 @@ files:
 - lib/eps/metrics.rb
 - lib/eps/model.rb
 - lib/eps/naive_bayes.rb
-- lib/eps/pmml_generators/lightgbm.rb
+- lib/eps/pmml.rb
+- lib/eps/pmml/generator.rb
+- lib/eps/pmml/loader.rb
 - lib/eps/statistics.rb
 - lib/eps/text_encoder.rb
 - lib/eps/utils.rb
@@ -141,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Machine learning for Ruby. Supports regression (linear regression) and classification

data/lib/eps/pmml_generators/lightgbm.rb

@@ -1,187 +0,0 @@
-module Eps
-  module PmmlGenerators
-    module LightGBM
-      private
-
-      def generate_pmml
-        feature_importance = @feature_importance
-
-        data_fields = {}
-        data_fields[@target] = @labels if @labels
-        @features.each_with_index do |(k, type), i|
-          # TODO remove zero importance features
-          if type == "categorical"
-            data_fields[k] = @label_encoders[k].labels.keys
-          else
-            data_fields[k] = nil
-          end
-        end
-
-        build_pmml(data_fields) do |xml|
-          function_name = @objective == "regression" ? "regression" : "classification"
-          xml.MiningModel(functionName: function_name, algorithmName: "LightGBM") do
-            xml.MiningSchema do
-              xml.MiningField(name: @target, usageType: "target")
-              @features.keys.each_with_index do |k, i|
-                # next if feature_importance[i] == 0
-                # TODO add importance, but need to handle text features
-                xml.MiningField(name: k) #, importance: feature_importance[i].to_f, missingValueTreatment: "asIs")
-              end
-            end
-            pmml_local_transformations(xml)
-
-            case @objective
-            when "regression"
-              xml_segmentation(xml, @trees)
-            when "binary"
-              xml.Segmentation(multipleModelMethod: "modelChain") do
-                xml.Segment(id: 1) do
-                  xml.True
-                  xml.MiningModel(functionName: "regression") do
-                    xml.MiningSchema do
-                      @features.each do |k, _|
-                        xml.MiningField(name: k)
-                      end
-                    end
-                    xml.Output do
-                      xml.OutputField(name: "lgbmValue", optype: "continuous", dataType: "double", feature: "predictedValue", isFinalResult: false) do
-                        xml.Apply(function: "/") do
-                          xml.Constant(dataType: "double") do
-                            1.0
-                          end
-                          xml.Apply(function: "+") do
-                            xml.Constant(dataType: "double") do
-                              1.0
-                            end
-                            xml.Apply(function: "exp") do
-                              xml.Apply(function: "*") do
-                                xml.Constant(dataType: "double") do
-                                  -1.0
-                                end
-                                xml.FieldRef(field: "lgbmValue")
-                              end
-                            end
-                          end
-                        end
-                      end
-                    end
-                    xml_segmentation(xml, @trees)
-                  end
-                end
-                xml.Segment(id: 2) do
-                  xml.True
-                  xml.RegressionModel(functionName: "classification", normalizationMethod: "none") do
-                    xml.MiningSchema do
-                      xml.MiningField(name: @target, usageType: "target")
-                      xml.MiningField(name: "transformedLgbmValue")
-                    end
-                    xml.Output do
-                      @labels.each do |label|
-                        xml.OutputField(name: "probability(#{label})", optype: "continuous", dataType: "double", feature: "probability", value: label)
-                      end
-                    end
-                    xml.RegressionTable(intercept: 0.0, targetCategory: @labels.last) do
-                      xml.NumericPredictor(name: "transformedLgbmValue", coefficient: "1.0")
-                    end
-                    xml.RegressionTable(intercept: 0.0, targetCategory: @labels.first)
-                  end
-                end
-              end
-            else # multiclass
-              xml.Segmentation(multipleModelMethod: "modelChain") do
-                n = @trees.size / @labels.size
-                @trees.each_slice(n).each_with_index do |trees, idx|
-                  xml.Segment(id: idx + 1) do
-                    xml.True
-                    xml.MiningModel(functionName: "regression") do
-                      xml.MiningSchema do
-                        @features.each do |k, _|
-                          xml.MiningField(name: k)
-                        end
-                      end
-                      xml.Output do
-                        xml.OutputField(name: "lgbmValue(#{@labels[idx]})", optype: "continuous", dataType: "double", feature: "predictedValue", isFinalResult: false)
-                      end
-                      xml_segmentation(xml, trees)
-                    end
-                  end
-                end
-                xml.Segment(id: @labels.size + 1) do
-                  xml.True
-                  xml.RegressionModel(functionName: "classification", normalizationMethod: "softmax") do
-                    xml.MiningSchema do
-                      xml.MiningField(name: @target, usageType: "target")
-                      @labels.each do |label|
-                        xml.MiningField(name: "lgbmValue(#{label})")
-                      end
-                    end
-                    xml.Output do
-                      @labels.each do |label|
-                        xml.OutputField(name: "probability(#{label})", optype: "continuous", dataType: "double", feature: "probability", value: label)
-                      end
-                    end
-                    @labels.each do |label|
-                      xml.RegressionTable(intercept: 0.0, targetCategory: label) do
-                        xml.NumericPredictor(name: "lgbmValue(#{label})", coefficient: "1.0")
-                      end
-                    end
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-
-      def xml_segmentation(xml, trees)
-        xml.Segmentation(multipleModelMethod: "sum") do
-          trees.each_with_index do |node, i|
-            xml.Segment(id: i + 1) do
-              xml.True
-              xml.TreeModel(functionName: "regression", missingValueStrategy: "none", noTrueChildStrategy: "returnLastPrediction", splitCharacteristic: "multiSplit") do
-                xml.MiningSchema do
-                  node_fields(node).uniq.each do |k|
-                    xml.MiningField(name: display_field(k))
-                  end
-                end
-                node_pmml(node, xml)
-              end
-            end
-          end
-        end
-      end
-
-      def node_fields(node)
-        fields = []
-        fields << node.field if node.predicate
-        node.children.each do |n|
-          fields.concat(node_fields(n))
-        end
-        fields
-      end
-
-      def node_pmml(node, xml)
-        xml.Node(score: node.score) do
-          if node.predicate.nil?
-            xml.True
-          elsif node.operator == "in"
-            xml.SimpleSetPredicate(field: display_field(node.field), booleanOperator: "isIn") do
-              xml.Array(type: "string") do
-                xml.text node.value.map { |v| escape_element(v) }.join(" ")
-              end
-            end
-          else
-            xml.SimplePredicate(field: display_field(node.field), operator: node.operator, value: node.value)
-          end
-          node.children.each do |n|
-            node_pmml(n, xml)
-          end
-        end
-      end
-
-      def escape_element(v)
-        "\"#{v.gsub("\"", "\\\"")}\""
-      end
-    end
-  end
-end