eps 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1369016c3cae228f169fe580b54fca3c0d240cda202fa7d03ecc7a4e156ee8c7
-  data.tar.gz: bf83ca424c509798d1a1436806b52cba0cfdbefecb8d827d5b17aec7b807b121
+  metadata.gz: d56573908e892d8d1959d66c7b6f2940f8930a2d0f2dfd5d4da75e2ff7cfdb63
+  data.tar.gz: 9eaf1a06c8c51ba15d9b4468796fc869f2933945494d027b54789304080c5d5b
 SHA512:
-  metadata.gz: 2bf47d80a301eb546c348aaa71f847fa22ace5bed63d97a1f19eb14bc15388b056cd3f545ccf251b2bbf2afc485ef81e5559849ff7459e9dd9f88a71c7cbf83a
-  data.tar.gz: 82d65d84e95a6518cd132c2a42cdec20afd05c0013192941b59ee0edb524874d12b2dd9082dd89be1422872c88e827e031469e43b80336c48c7eab7ff4fe611e
+  metadata.gz: 971dbd2a95a280ed50925df68a29018ba7b3bccb7094b1374923a8ce7d100720202245843e003b26447832e9c1f8285bafcc7692020f5971a56c0a8e89a12afb
+  data.tar.gz: de06585dc75608b0f8c62188cce351987a0cd53f3b12889d4d63de28ed81ae1b143e31f47ac8c53083eeb250e18c5f8b721fff94a378e14203fd8fa90ba3e440

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.3.6 (2020-06-19)
+
+- Fixed error with text features for LightGBM
+
 ## 0.3.5 (2020-06-10)
 
 - Added `learning_rate` option for LightGBM

data/README.md

@@ -343,9 +343,7 @@ df = Daru::DataFrame.from_csv("houses.csv")
 Eps::Model.new(df, target: "price")
 ```
 
-### CSVs
-
-When importing data from CSV files, be sure to convert numeric fields. The `table` method does this automatically.
+When reading CSV files directly, be sure to convert numeric fields. The `table` method does this automatically.
 
 ```ruby
 CSV.table("data.csv").map { |row| row.to_h }
@@ -375,7 +373,11 @@ Eps::Model.new(data, learning_rate: 0.01)
 
 ### Linear Regression
 
-#### Performance
+By default, an intercept is included. Disable this with:
+
+```ruby
+Eps::Model.new(data, intercept: false)
+```
 
 To speed up training on large datasets with linear regression, [install GSL](https://github.com/ankane/gslr#gsl-installation). With Homebrew, you can use:
 
@@ -391,14 +393,6 @@ gem 'gslr', group: :development
 
 It only needs to be available in environments used to build the model.
 
-#### Options
-
-By default, an intercept is included. Disable this with:
-
-```ruby
-Eps::Model.new(data, intercept: false)
-```
-
 ## Probability
 
 To get the probability of each category for predictions with classification, use:

data/lib/eps/base_estimator.rb

@@ -3,6 +3,7 @@ module Eps
     def initialize(data = nil, y = nil, **options)
       @options = options.dup
       @trained = false
+      @text_encoders = {}
       # TODO better pattern - don't pass most options to train
       train(data, y, **options) if data
     end
@@ -209,29 +210,38 @@ module Eps
       [data, target]
     end
 
-    def prep_text_features(train_set)
-      @text_encoders = {}
+    def prep_text_features(train_set, fit: true)
       @text_features.each do |k, v|
-        # reset vocabulary
-        v.delete(:vocabulary)
+        if fit
+          # reset vocabulary
+          v.delete(:vocabulary)
+
+          # TODO determine max features automatically
+          # start based on number of rows
+          encoder = Eps::TextEncoder.new(**v)
+          counts = encoder.fit(train_set.columns.delete(k))
+        else
+          encoder = @text_encoders[k]
+          counts = encoder.transform(train_set.columns.delete(k))
+        end
 
-        # TODO determine max features automatically
-        # start based on number of rows
-        encoder = Eps::TextEncoder.new(**v)
-        counts = encoder.fit(train_set.columns.delete(k))
         encoder.vocabulary.each do |word|
           train_set.columns[[k, word]] = [0] * counts.size
         end
+
         counts.each_with_index do |ci, i|
           ci.each do |word, count|
             word_key = [k, word]
             train_set.columns[word_key][i] = 1 if train_set.columns.key?(word_key)
           end
         end
-        @text_encoders[k] = encoder
 
-        # update vocabulary
-        v[:vocabulary] = encoder.vocabulary
+        if fit
+          @text_encoders[k] = encoder
+
+          # update vocabulary
+          v[:vocabulary] = encoder.vocabulary
+        end
       end
 
       raise "No features left" if train_set.columns.empty?

data/lib/eps/evaluators/lightgbm.rb

@@ -19,13 +19,7 @@ module Eps
         # sparse matrix
         @text_features.each do |k, v|
           encoder = TextEncoder.new(**v)
-
-          values = data.columns.delete(k)
-          counts = encoder.transform(values)
-
-          encoder.vocabulary.each do |word|
-            data.columns[[k, word]] = [0] * values.size
-          end
+          counts = encoder.transform(data.columns[k])
 
           counts.each_with_index do |xc, i|
             row = rows[i]
@@ -52,7 +46,7 @@ module Eps
           @trees.each_slice(num_trees).each do |trees|
             tree_scores << sum_trees(rows, trees)
           end
-          data.size.times.map do |i|
+          rows.size.times.map do |i|
             v = tree_scores.map { |s| s[i] }
             if probabilities
               exp = v.map { |vi| Math.exp(vi) }

data/lib/eps/lightgbm.rb

@@ -57,7 +57,7 @@ module Eps
 
       # text feature encoding
       prep_text_features(train_set)
-      prep_text_features(validation_set) if validation_set
+      prep_text_features(validation_set, fit: false) if validation_set
 
       # create params
       params = {
@@ -144,7 +144,10 @@ module Eps
       end
 
       if bad_observations.any?
-        raise "Bug detected in evaluator. Please report an issue. Bad data points: #{bad_observations.inspect}"
+        bad_observations.each do |obs|
+          p obs
+        end
+        raise "Bug detected in evaluator. Please report an issue."
       end
     end
 

data/lib/eps/version.rb

@@ -1,3 +1,3 @@
 module Eps
-  VERSION = "0.3.5"
+  VERSION = "0.3.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eps
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-11 00:00:00.000000000 Z
+date: 2020-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: lightgbm