eps 0.3.5 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1369016c3cae228f169fe580b54fca3c0d240cda202fa7d03ecc7a4e156ee8c7
-  data.tar.gz: bf83ca424c509798d1a1436806b52cba0cfdbefecb8d827d5b17aec7b807b121
+  metadata.gz: 49da7dab8a04a1404446f56051ea9faaf5b2260888ef9e2eccf12ec0f5ece887
+  data.tar.gz: 56a85a477e48cb07b39253e182878a947371ec30375b4570987c885a0e6bb93a
 SHA512:
-  metadata.gz: 2bf47d80a301eb546c348aaa71f847fa22ace5bed63d97a1f19eb14bc15388b056cd3f545ccf251b2bbf2afc485ef81e5559849ff7459e9dd9f88a71c7cbf83a
-  data.tar.gz: 82d65d84e95a6518cd132c2a42cdec20afd05c0013192941b59ee0edb524874d12b2dd9082dd89be1422872c88e827e031469e43b80336c48c7eab7ff4fe611e
+  metadata.gz: 4a611604c4172110f67e3b39147ddb77346fe98390ee9a31791f61e1da2b4a6f1f6e83f63c4ab722c30d2247797c66328b9acefb51a907f76808793014207dd2
+  data.tar.gz: 1aca1429a26a1eafee5717680507cf04794f639bb13c4ce416709bf0e64ca6017525413f74bcfa7728bd5272ff5b845dfb2da17cdee781eb03ea98a56c3441b3

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 0.3.9 (2021-10-14)
+
+- Fixed error with `lessOrEqual` operator
+
+## 0.3.8 (2021-02-08)
+
+- Fixed error with categorical and text features
+
+## 0.3.7 (2020-11-23)
+
+- Fixed error with LightGBM summary
+
+## 0.3.6 (2020-06-19)
+
+- Fixed error with text features for LightGBM
+
 ## 0.3.5 (2020-06-10)
 
 - Added `learning_rate` option for LightGBM

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018-2019 Andrew Kane
+Copyright (c) 2018-2021 Andrew Kane
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -7,7 +7,7 @@ Machine learning for Ruby
 
 Check out [this post](https://ankane.org/rails-meet-data-science) for more info on machine learning with Rails
 
-[![Build Status](https://travis-ci.org/ankane/eps.svg?branch=master)](https://travis-ci.org/ankane/eps)
+[![Build Status](https://github.com/ankane/eps/workflows/build/badge.svg?branch=master)](https://github.com/ankane/eps/actions)
 
 ## Installation
 
@@ -134,7 +134,7 @@ For text features, use strings with multiple words.
 {description: "a beautiful house on top of a hill"}
 ```
 
-This creates features based on word count (term frequency).
+This creates features based on [word count](https://en.wikipedia.org/wiki/Bag-of-words_model).
 
 You can specify text features explicitly with:
 
@@ -147,12 +147,12 @@ You can set advanced options with:
 ```ruby
 text_features: {
   description: {
-    min_occurences: 5,
-    max_features: 1000,
-    min_length: 1,
-    case_sensitive: true,
-    tokenizer: /\s+/,
-    stop_words: ["and", "the"]
+    min_occurences: 5,          # min times a word must appear to be included in the model
+    max_features: 1000,         # max number of words to include in the model
+    min_length: 1,              # min length of words to be included
+    case_sensitive: true,       # how to treat words with different case
+    tokenizer: /\s+/,           # how to tokenize the text, defaults to whitespace
+    stop_words: ["and", "the"]  # words to exclude from the model
   }
 }
 ```
@@ -218,7 +218,7 @@ Build the model with:
 PriceModel.build
 ```
 
-This saves the model to `price_model.pmml`. Be sure to check this into source control.
+This saves the model to `price_model.pmml`. Check this into source control or use a tool like [Trove](https://github.com/ankane/trove) to store it.
 
 Predict with:
 
@@ -343,9 +343,7 @@ df = Daru::DataFrame.from_csv("houses.csv")
 Eps::Model.new(df, target: "price")
 ```
 
-### CSVs
-
-When importing data from CSV files, be sure to convert numeric fields. The `table` method does this automatically.
+When reading CSV files directly, be sure to convert numeric fields. The `table` method does this automatically.
 
 ```ruby
 CSV.table("data.csv").map { |row| row.to_h }
@@ -375,7 +373,11 @@ Eps::Model.new(data, learning_rate: 0.01)
 
 ### Linear Regression
 
-#### Performance
+By default, an intercept is included. Disable this with:
+
+```ruby
+Eps::Model.new(data, intercept: false)
+```
 
 To speed up training on large datasets with linear regression, [install GSL](https://github.com/ankane/gslr#gsl-installation). With Homebrew, you can use:
 
@@ -391,14 +393,6 @@ gem 'gslr', group: :development
 
 It only needs to be available in environments used to build the model.
 
-#### Options
-
-By default, an intercept is included. Disable this with:
-
-```ruby
-Eps::Model.new(data, intercept: false)
-```
-
 ## Probability
 
 To get the probability of each category for predictions with classification, use:

data/lib/eps/base_estimator.rb

@@ -3,6 +3,7 @@ module Eps
     def initialize(data = nil, y = nil, **options)
       @options = options.dup
       @trained = false
+      @text_encoders = {}
       # TODO better pattern - don't pass most options to train
       train(data, y, **options) if data
     end
@@ -209,29 +210,38 @@ module Eps
       [data, target]
     end
 
-    def prep_text_features(train_set)
-      @text_encoders = {}
+    def prep_text_features(train_set, fit: true)
       @text_features.each do |k, v|
-        # reset vocabulary
-        v.delete(:vocabulary)
+        if fit
+          # reset vocabulary
+          v.delete(:vocabulary)
+
+          # TODO determine max features automatically
+          # start based on number of rows
+          encoder = Eps::TextEncoder.new(**v)
+          counts = encoder.fit(train_set.columns.delete(k))
+        else
+          encoder = @text_encoders[k]
+          counts = encoder.transform(train_set.columns.delete(k))
+        end
 
-        # TODO determine max features automatically
-        # start based on number of rows
-        encoder = Eps::TextEncoder.new(**v)
-        counts = encoder.fit(train_set.columns.delete(k))
         encoder.vocabulary.each do |word|
           train_set.columns[[k, word]] = [0] * counts.size
         end
+
         counts.each_with_index do |ci, i|
           ci.each do |word, count|
             word_key = [k, word]
             train_set.columns[word_key][i] = 1 if train_set.columns.key?(word_key)
           end
         end
-        @text_encoders[k] = encoder
 
-        # update vocabulary
-        v[:vocabulary] = encoder.vocabulary
+        if fit
+          @text_encoders[k] = encoder
+
+          # update vocabulary
+          v[:vocabulary] = encoder.vocabulary
+        end
       end
 
       raise "No features left" if train_set.columns.empty?

data/lib/eps/evaluators/lightgbm.rb

@@ -19,13 +19,7 @@ module Eps
         # sparse matrix
         @text_features.each do |k, v|
           encoder = TextEncoder.new(**v)
-
-          values = data.columns.delete(k)
-          counts = encoder.transform(values)
-
-          encoder.vocabulary.each do |word|
-            data.columns[[k, word]] = [0] * values.size
-          end
+          counts = encoder.transform(data.columns[k])
 
           counts.each_with_index do |xc, i|
             row = rows[i]
@@ -52,7 +46,7 @@ module Eps
           @trees.each_slice(num_trees).each do |trees|
             tree_scores << sum_trees(rows, trees)
           end
-          data.size.times.map do |i|
+          rows.size.times.map do |i|
             v = tree_scores.map { |s| s[i] }
             if probabilities
               exp = v.map { |vi| Math.exp(vi) }

data/lib/eps/lightgbm.rb

@@ -10,7 +10,7 @@ module Eps
         str << "Model needs more data for better predictions\n"
       else
         str << "Most important features\n"
-        @importance_keys.zip(importance).sort_by { |k, v| [-v, k] }.first(10).each do |k, v|
+        @importance_keys.zip(importance).sort_by { |k, v| [-v, display_field(k)] }.first(10).each do |k, v|
           str << "#{display_field(k)}: #{(100 * v / total).round}\n"
         end
       end
@@ -57,7 +57,7 @@ module Eps
 
       # text feature encoding
       prep_text_features(train_set)
-      prep_text_features(validation_set) if validation_set
+      prep_text_features(validation_set, fit: false) if validation_set
 
       # create params
       params = {
@@ -71,7 +71,7 @@ module Eps
       end
 
       # create datasets
-      categorical_idx = @features.values.map.with_index.select { |type, _| type == "categorical" }.map(&:last)
+      categorical_idx = train_set.columns.keys.map.with_index.select { |k, _| @features[k] == "categorical" }.map(&:last)
       train_ds = ::LightGBM::Dataset.new(train_set.map_rows(&:to_a), label: train_set.label, weight: train_set.weight, categorical_feature: categorical_idx, params: params)
       validation_ds = ::LightGBM::Dataset.new(validation_set.map_rows(&:to_a), label: validation_set.label, weight: validation_set.weight, categorical_feature: categorical_idx, params: params, reference: train_ds) if validation_set
 
@@ -144,7 +144,10 @@ module Eps
       end
 
       if bad_observations.any?
-        raise "Bug detected in evaluator. Please report an issue. Bad data points: #{bad_observations.inspect}"
+        bad_observations.each do |obs|
+          p obs
+        end
+        raise "Bug detected in evaluator. Please report an issue."
       end
     end
 

data/lib/eps/pmml/loader.rb

@@ -222,7 +222,7 @@ module Eps
           else
             operator = xml_predicate.attribute("operator").value
             value = xml_predicate.attribute("value").value
-            value = value.to_f if operator == "greaterThan"
+            value = value.to_f if operator == "greaterThan" || operator == "lessOrEqual"
             field = xml_predicate.attribute("field").value
             field = derived_fields[field] if derived_fields[field]
             {

data/lib/eps/version.rb

@@ -1,3 +1,3 @@
 module Eps
-  VERSION = "0.3.5"
+  VERSION = "0.3.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eps
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.9
 platform: ruby
 authors:
 - Andrew Kane
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-11 00:00:00.000000000 Z
+date: 2021-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: lightgbm
@@ -38,92 +38,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: daru
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: numo-narray
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rover-df
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: 
-email: andrew@chartkick.com
+description:
+email: andrew@ankane.org
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -156,7 +72,7 @@ homepage: https://github.com/ankane/eps
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -171,8 +87,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: Machine learning for Ruby. Supports regression (linear regression) and classification
   (naive Bayes)