clarifier 0.0.3 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d26cc44221f2412fcd48d2463a2f306a6d8a5ea7
-  data.tar.gz: 333525a5ca5934d42b2c69cb3c7901be9b8fc765
+  metadata.gz: a284e14c6ff8275e044ef65de1103c50f904b015
+  data.tar.gz: 89c5f5b8d614acd0b02efe2bb4c6fa117f225cc5
 SHA512:
-  metadata.gz: 58533ded8c5c7bd74310a6db0eed6f082136288bd19d1e2627010df201057435344495526cc8683fee45e9f7ad206ca2f3bea6a74337a9561e46e781f967384d
-  data.tar.gz: 41ded05e9f3e2b4fba1fb628946e2154ae98e719e125df36e5dff22475b73a9aeadff54f9d0212c33eda574451129079ec8ad4c6c3f359c564ec421cb17e0450
+  metadata.gz: fabd1582bfe6cf87592e7c5c4bf0264beb3f02b7d173f0716033ad7908edd1ba9d8e88ee323f1fe818ba54ca201df14d62df4f12fec6da945cd74d3a802b172d
+  data.tar.gz: a337256cf75701be19943bfdf21217baecff51e1100096b1f03a98582f117df8bd76ad5e5a69b5d6875a247645dfff6a9fc50b5cc66a05420b2bbd97011f222e

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 0.9.0 / 2014-12-18
+* [FEATURE] Incremental training of stopwords list using refine() and reset()
+
 # 0.0.3 / 2014-04-07
 * [BUGFIX] Simplified regex
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    clarifier (0.0.3)
+    clarifier (0.9.0)
 
 GEM
   remote: https://rubygems.org/

data/lib/clarifier/stop_words.rb

@@ -9,7 +9,7 @@ module Clarifier
       @@lists
     end
 
-    def initialize(desired_stopwords = nil)
+    def initialize(desired_stopwords = nil, training_threshold = 0.8)
       if desired_stopwords.kind_of?(Array)
         @stopwords = desired_stopwords
       elsif @@lists[desired_stopwords]
@@ -17,6 +17,7 @@ module Clarifier
       else
         @stopwords = @@lists[:en_gb_basic]
       end
+      @training_threshold = training_threshold
     end
 
     def clarify(input)
@@ -32,18 +33,37 @@ module Clarifier
       new_string
     end
 
-    def train(docs, threshold = 0.8)
-      word_counts = {}
+    def reset
+      @word_counts = Hash.new(0)
+      @training_doc_count = 0
+      @stopwords = []
+    end
+
+    def refine(doc)
+      @word_counts ||= Hash.new(0)
+      @training_doc_count ||= 0
+      @training_doc_count += 1
+      words = doc.split
+      words.uniq!
+      words.each do |word|
+        @word_counts[word] += 1
+      end
+      select_stopwords_from_training
+    end
+
+    def train(docs, threshold = @training_threshold)
+      @word_counts = Hash.new(0)
+      @training_doc_count = 0
+      @training_threshold = threshold
       docs.each do |doc|
-        words = doc.split
-        words.uniq!
-        words.each do |word|
-          word_counts[word] ||= 0
-          word_counts[word] += 1
-        end
+        refine(doc)
       end
-      word_counts.each do |word,count|
-        if count.to_f / docs.length >= threshold
+    end
+
+    def select_stopwords_from_training
+      @stopwords = []
+      @word_counts.each do |word, count|
+        if count.to_f / @training_doc_count >= @training_threshold
           @stopwords << word
         end
       end

data/lib/clarifier/version.rb

@@ -1,3 +1,3 @@
 module Clarifier
-  VERSION = "0.0.3"
+  VERSION = "0.9.0"
 end

data/test/stop_words_test.rb

@@ -45,7 +45,14 @@ module Clarifier
     end
 
     def test_incrementally_derive_a_stopword_list_from_docs
-      skip
+      sw = Clarifier::StopWords.new([], 0.2)
+      StopWordsTestDocs.docs.each do |doc|
+        sw.refine(doc)
+      end
+      expected = %w(today we are going to discuss about the heart disease first for a run of concept and then we'll see what is between but at actually three concepts number one should special type whole time with people magic this an class workers that have up infection mission let's take our book bad give here or really back in which end bikini can inside them does especially by let me tell you particular produced who it don't know occurs because these dot part i when grow on its free political so hard out air your into that's true indeed talking right now has be across done must not go girl did if more supreme some over pretty friend like as bed system next their it's working think will start doesn't off there area present from good they stop him those diagram news red blood cell size cells regular he caused either environment two examples even pick way his side years easy however test many again chronic body were peers six having work tonight defined high loss jeans term process involves nothing help yeah get put world getting I'm waking doing us how lives living I do arm almost things same such close life hands you're just possible they're gonna happen down choose other rest changed tend little bit want car look mom looking kids kinda each she's got leg trying very quickly thing anything sorta yet case use only nineteen seventy medical been school medicine center health access patients proud last we're near new eighty nine group question towards complicated sometimes called able consistent too goes different both say he's all bodies physiology personal eyes no person until past within conditions would point come levels identify essential court well state act another against sort own view i'll refer make important under sir challenge face understanding uh... might began critical much sixty minutes sells understand week talk study hopefully lecture why won't live long without normal kidney chapter gross aspects differences fluid thought talked was created lot quite means show may block blue box around you'll remember low cast nerve signal four cardiac muscle contract yes action brains am could knowing always keep difficult general excess somehow sure left ask after pass basically said involved my young become fine spreading I'll through change maybe okay research most where buildings sections example find terrible cold coast something seeing moving mention boy fibers clearly underneath nuclei middle better try nice ah... took closely also bridge jason contraction ever ice had top trailer behind press didn't realize later seven brain hook surgery spinal cord)
+      assert_equal expected, sw.stopwords
+      sw.reset
+      assert_equal [], sw.stopwords
     end
 
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: clarifier
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.9.0
 platform: ruby
 authors:
 - Rob Styles
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-07 00:00:00.000000000 Z
+date: 2014-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -111,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Clarifier is a stopwords library for removing common words from text