huginn_naive_bayes_agent 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c4bf23b64717fc62020b16dcc96b3a9767e71f42
-  data.tar.gz: d3687649e9758f6f57e71478ec4edeb47c9d4c1e
+  metadata.gz: 7587d0f8882b59f15113b585cfb259bc14c8ca1e
+  data.tar.gz: be9c5416f658807eadff0e7deeeaae28a9260d65
 SHA512:
-  metadata.gz: 97a60c81731ae167e119b8bb68852341f15d7d5e1d6dd03ce728662a9adcdbe42ca5b2886dcfa8ba9e3c4cfe9186fd6f621f916dc3a303cd7a24c29182bcaf27
-  data.tar.gz: 86bc9dfe3906f16cac1851bc38257768ded8fbb7611a527c35946e19ac03939dd61e511e4056c44cc4171331be44f6c244fe13081c34fd2d18c37757c60c746b
+  metadata.gz: ab9ec982cc1bf2a5bc49001069b25bc10db65654c57edd591d8e6d446e802498505c499f78680b2b89afe6a16351792d5a06060e66b76cd205fb4dfc6f390bc0
+  data.tar.gz: bcd43b5588a809d542bbaefbfc962508bb90f98737f33e53673aea3cef56202e66f2643101c3fb3a427e60653b27138d8c04e11722e39cdfa2d507ce34ce1648

data/lib/huginn_naive_bayes_agent/naive_bayes_agent.rb

@@ -1,5 +1,6 @@
 require 'nbayes'
-require'yaml'
+require 'yaml'
+require 'fast_stemmer'
 
 module Agents
   class NaiveBayesAgent < Agent
@@ -18,7 +19,9 @@ module Agents
       
       However, if `nb_cats` is already populated, then the content from `nb_content` will be used as training data for the categories listed in `nb_cats`. For instance, say `nb_cats` consists of `trees`. Then `nb_content` will be used as training data for the category `trees`. The data is saved to the agent memory. 
       
-      Data in `nb_content` can be cleaned before classification. If `strip_punctuation` is set to true, the text in `nb_content` is stripped of punctuation before it is sent to the classifier. The changes are not saved to `nb_content`.
+      Data in `nb_content` can be cleaned before classification. If `strip_punctuation` is set to true, the text in `nb_content` is stripped of punctuation before it is sent to the classifier. The changes are not saved to `nb_content` but will affect the Agent's saved training data.
+      
+      Content can also be "stemmed", reducing words to their base, by setting `stem` to true. Stemming will reduce "epistemology", "epistemologies", and "epistemological" all to "epistemolog". See [here](https://github.com/romanbsd/fast-stemmer) for the implementation used. Again, changes are not saved to `nb_content` but will affect the Agent's saved training data.
             
       When an event is received for classification, the Naive Bayes Agent will assign a value between 0 and 1 representing the likelihood that it falls under a category. The `min_value` option lets you choose the minimum threshold that must be reached before the event is labeled with that category. If `min_value` is set to 1, then the event is labeled with whichever category has the highest value. 
       
@@ -26,7 +29,7 @@ module Agents
       
       To load trained data into an agent's memory, create a Manual Agent with `nb_cats : =loadYML` and `nb_content : your-well-formed-training-data-here`. Use the text input box, not the form view, by clicking "Toggle View" when inputting your training data else whitespace errors occur in the YML. Then submit this to your Naive Bayes Agent.
       
-      #### Advanced Features 
+      #### Advanced Naive Bayes Features 
       
       ##### Only works if the nbayes dependency was installed from Github, version => .1.2. Rubygems is still .1.1
       
@@ -46,7 +49,8 @@ module Agents
         'min_value' => "0.5",
         'propagate_training_events' => 'true',
         'expected_update_period_in_days' => "7",
-        'strip_punctuation' => 'false'
+        'strip_punctuation' => 'false',
+        'stem' => 'false'
       }
     end
 
@@ -90,6 +94,9 @@ module Agents
             if interpolated['strip_punctuation'] == "true"
               nb_content = nb_content.gsub(/[^[:word:]\s]/, '') #https://stackoverflow.com/a/10074271
             end
+            if interpolated['stem'] == "true"
+              nb_content = nb_content.split(/\s+/).map{|word| word.stem}.join(" ")
+            end
             cats.each do |c|
               c.starts_with?('-') ? nbayes.untrain(nb_content.split(/\s+/), c[1..-1]) : nbayes.train(nb_content.split(/\s+/), c)
             end
@@ -104,6 +111,9 @@ module Agents
           if interpolated['strip_punctuation'] == "true"
             nb_content = nb_content.gsub(/[^[:word:]\s]/, '') #https://stackoverflow.com/a/10074271
           end
+          if interpolated['stem'] == "true"
+            nb_content = nb_content.split(/\s+/).map{|word| word.stem}.join(" ")
+          end
           result = nbayes.classify(nb_content.split(/\s+/))
           if interpolated['min_value'].to_f == 1
             event.payload['nb_cats'] << (event.payload['nb_cats'].length == 0 ? result.max_class : " "+result.max_class)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: huginn_naive_bayes_agent
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Noah Greenstein
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fast-stemmer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: The Huginn Naive Bayes agent uses some incoming Events as a training
   set for Naive Bayes Machine Learning. Then it classifies Events from other sources
   accordingly using tags. Acts as a Huginn Agent front end to the NBayes gem (https://github.com/oasic/nbayes).