te_rex 0.0.10 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 41cbaf3893f705db1f366bdf247e0882971d860b
-  data.tar.gz: 909c02dd6578c76bdc5ec7ae4cab8f3b7cedd8d1
+  metadata.gz: 929e90be95279fcf5093f8d865457509bcc5e692
+  data.tar.gz: 98e1e7ad046d7e3f423934187b0df0546a6500a0
 SHA512:
-  metadata.gz: b9878b1e86ec13e17dd892abc4c615e1e369f692ea020413f121687af8b70c50a597557f7f36674f2a87464657edd62edd9f4ca62ab901fe3e042d98443180b8
-  data.tar.gz: 08a16cf36d77697253a56147fc3903f505ef5c09ebbfa7eb6f03474e44f87418a29a8c4640c8a46fb242cc441a39e193fa57bb9a8218653095eec9b3c519451e
+  metadata.gz: ad648071a76d79757ecdbc793a31753bdfdf94f6c0e8b462b9f315c8d514e9fdc07c942d2adae49dc20c7d35911554ff48c7f41c6804cf62c1727c16542d897a
+  data.tar.gz: a6397c1ea3b21735813c8f2cd739b889720d08746a5ea1903db1976890ea2c9eaac2e316ea886b2d6305c1e8ef3e02053c560b62dacaa3cfc5f0390094989893

data/lib/format/{corpus/basic_file.rb → basic_file.rb}

@@ -2,7 +2,7 @@ module TeRex
   module Format
     class BasicFile
 
-      attr_accessor :sentences
+      attr_reader :sentences, :path
 
       def initialize(file_path)
         @path = file_path
@@ -22,8 +22,8 @@ module TeRex
             end
           end
         end
-
       end
+
     end
   end
 end

data/lib/format/error_file.rb

@@ -0,0 +1,29 @@
+module TeRex
+  module Format
+    require 'csv'
+    class ErrorFile
+
+      attr_reader :sentences, :path
+
+      @@csv_conf = {:headers => true}
+
+      def initialize(file_path)
+        @path = file_path
+      end
+
+      # Each row of csv as Array object, strip it and return 
+      def scanner
+        accumulator = []
+        CSV.foreach(@path, @@csv_conf) do |row|
+          next if row.empty?
+          stripped_line = row[0].strip
+          unless stripped_line.nil? || stripped_line.empty?
+            accumulator << stripped_line
+          end
+        end
+        @sentences ||= accumulator
+      end
+
+    end
+  end
+end

data/lib/te_rex/corpus.rb

@@ -2,39 +2,88 @@ module TeRex
   module Corpus
     class Body
 
-      attr_accessor :files, :sample_size, :training, :testing
+      attr_reader :set, :sample_size, :training, :testing, :format_klass, :category_klass, :total_sentences
 
-      def initialize(glob, klass)
+      def initialize(glob: "", partition: :file, format_klass: NilClass, category_klass: NilClass)
         @glob = glob
-        @klass = klass
+        @format_klass = format_klass
+        @category_klass = category_klass
+        @partition = partition
       end
 
+      #@sample_size = (@set.count * 0.75).round
       def build
-        get_files
-        @training_sentences = partition_train
-        @testing_sentences = partition_test
+        define_set
+        case @partition
+        when /file/
+          file_partition
+        else
+          sentence_partition
+        end
+      end
+
+      def file_partition
+        @sample_size = (@set.count.to_f * 0.75).round
+        @training = partition_training_by_file
+        @testing = partition_test_by_file
+        count_all
+      end
+
+      def sentence_partition
+        #super_set = build_superset
+        corpus_set = partition_files_for_sentences
+        #@sample_size = (superset.count.to_f * 0.75).round
+        @sample_size = 0.0
+        @training = partition_training_by_sentence(corpus_set)
+        @testing = partition_test_by_sentence(corpus_set)
+        count_all
       end
 
-      def get_files
-        @files ||= Dir[@glob].map do |file|
-          @klass.new(file)
+      def define_set
+        @set ||= Dir[@glob].map do |file|
+          @format_klass.new(file)
         end
-        @sample_size = (@files.count * 0.75).round
-        @files
+        @set
+      end
+
+      def partition_training_by_file
+        @set[0..@sample_size].map do |file|
+          file.scanner
+        end.flatten
       end
 
-      def partition_train
-        @files[0..@sample_size].map do |file|
+      def partition_test_by_file
+        @set[(@sample_size - 1)..-1].map do |file|
           file.scanner
         end.flatten
       end
 
-      def partition_test
-        @files[(@sample_size - 1)..-1].map do |file|
+      def partition_files_for_sentences
+        @set.map do |file|
           file.scanner
         end.flatten
       end
 
+      def partition_training_by_sentence(c_set)
+        c_set.sample(c_set.count * 0.75)
+      end
+
+      def partition_test_by_sentence(c_set)
+        c_set.sample(c_set.count * 0.25)
+      end
+
+      def build_superset
+        @set.reduce([]) do |memo,formatter|
+          memo << formatter.sentences
+        end.flatten
+      end
+
+      def count_all
+        counter = 0
+        @set.map{|f| counter += f.sentences.count}
+        @total_sentences = counter
+      end
+
     end
   end
 end

data/lib/te_rex/version.rb

@@ -1,3 +1,3 @@
 module TeRex
-  VERSION = "0.0.10"
+  VERSION = "0.0.11"
 end

data/lib/te_rex.rb

@@ -6,8 +6,9 @@
 #end
 
 require_relative "format/format"
-require_relative "format/corpus/brown_file"
-require_relative "format/corpus/basic_file"
+require_relative "format/brown_file"
+require_relative "format/basic_file"
+require_relative "format/error_file"
 require_relative "te_rex/stop_word"
 require_relative "te_rex/alpha_num"
 require_relative "te_rex/bayes_data"

data/test/bayes_data_test.rb

@@ -58,7 +58,7 @@ class BayesDataTest < MicroTest::Test
   end
 
   test "index frequency has correct counts" do
-    s = "Here is a sentence $141.34 that that $60 that 123.56 I need & & ^ % $c#@ to check the index is correct and okay."
+    s = 'Here is a sentence $141.34 that that $60 that 123.56 I need & & ^ % $c#@ to check the index is correct and okay.'
     result = TeRex::Classifier::BayesData.index_frequency(s)
 
     assert result[:moneyterm] == 3

data/test/corpus_test.rb

@@ -0,0 +1,57 @@
+#require_relative "../lib/te_rex"
+class CorpusTest < MicroTest::Test
+  class MockErrorClassifier
+  end
+
+  @@error_corpus = TeRex::Corpus::Body.new(glob: "test/test_modules/*.csv", format_klass: TeRex::Format::ErrorFile, category_klass: MockErrorClassifier)
+  @@error_corpus.build
+
+  test "Corpus has correct data before building" do
+    assert @@error_corpus.format_klass.name == "TeRex::Format::ErrorFile"
+    assert @@error_corpus.category_klass.name == "CorpusTest::MockErrorClassifier"
+  end
+
+  test "total count of sentences is correct" do
+    assert @@error_corpus.total_sentences == 12
+  end
+
+  test "ratio of training to testing is within 70%" do
+    ratio = @@error_corpus.testing.count.to_f / @@error_corpus.training.count.to_f
+    assert (60...80).map{|i| i}.include?((ratio * 100).to_i)
+  end
+
+  test "sentence counts are correct" do
+    assert @@error_corpus.set.count == 3
+    assert @@error_corpus.training.count == 12
+    assert @@error_corpus.testing.count == 8
+  end
+
+  @@sent_corpus = TeRex::Corpus::Body.new(glob: "test/test_modules/*.csv", partition: :sentence, format_klass: TeRex::Format::ErrorFile, category_klass: MockErrorClassifier)
+  @@sent_corpus.build
+
+  test "Corpus has correct data before building" do
+    assert @@sent_corpus.format_klass.name == "TeRex::Format::ErrorFile"
+    assert @@sent_corpus.category_klass.name == "CorpusTest::MockErrorClassifier"
+  end
+
+  test "total count of sentences is correct" do
+    assert @@sent_corpus.total_sentences == 12
+  end
+
+  test "ratio of training to total is about 75%" do
+    ratio = @@sent_corpus.training.count.to_f / @@sent_corpus.total_sentences
+    assert (72...77).map{|i| i}.include?((ratio * 100).to_i)
+  end
+
+  test "ratio of training to total is about 25%" do
+    ratio = @@sent_corpus.testing.count.to_f / @@sent_corpus.total_sentences
+    assert (22...27).map{|i| i}.include?((ratio * 100).to_i)
+  end
+
+  test "sentence counts are correct" do
+    assert @@sent_corpus.set.count == 3
+    assert @@sent_corpus.training.count == 9
+    assert @@sent_corpus.testing.count == 3
+  end
+
+end

data/test/sparse_bayes_test.rb

@@ -22,7 +22,12 @@ class SparseBayesTest < MicroTest::Test
     "The cancellation policy will be determined when the rate is validated."
   ]
 
-  @@cls = TeRex::Classifier::Bayes.new("Refund", "Partrefund", "Nonrefund", "Unknown")
+  @@cls = TeRex::Classifier::Bayes.new(
+    {:tag => "Refund",     :msg => "We are pleased to offer you a refund"},
+    {:tag => "Partrefund", :msg => "You may receive a partial refund"},
+    {:tag => "Nonrefund",  :msg => "Much apologies, no refund to you"},
+    {:tag => "Unknown",    :msg => "Waht?"}
+  )
   @@refund.each {|txt| @@cls.train("Refund", txt) }
   @@partrefund.each {|txt| @@cls.train("Partrefund", txt) }
   @@norefund.each {|txt| @@cls.train("Nonrefund", txt) }
@@ -40,15 +45,15 @@ class SparseBayesTest < MicroTest::Test
     s_non1 = @@cls.classify(s_non)
     s_unk1= @@cls.classify(s_unk)
 
-    assert s_refund1 == "Refund"
-    assert s_partial1 == "Partrefund"
-    assert s_non1 == "Nonrefund"
-    assert s_unk1 == "Unknown"
+    assert s_refund1 == ["Refund", "We are pleased to offer you a refund"]
+    assert s_partial1 == ["Partrefund", "You may receive a partial refund"]
+    assert s_non1 == ["Nonrefund", "Much apologies, no refund to you"]
+    assert s_unk1 == ["Unknown", "Waht?"]
 
-    assert s_refund1 != "Partrefund"
-    assert s_partial1 != "Refund"
-    assert s_non1 != "Unknown"
-    assert s_unk1 != "Nonrefund"
+    assert s_refund1 != ["Partrefund", "You may receive a partial refund"]
+    assert s_partial1 != ["Refund", "We are pleased to offer you a refund"]
+    assert s_non1 != ["Unknown", "Waht?"]
+    assert s_unk1 != ["Nonrefund", "Much apologies, no refund to you"]
   end
 
 
@@ -64,10 +69,10 @@ class SparseBayesTest < MicroTest::Test
     s33 = @@cls.classify(s3)
     s44 = @@cls.classify(s4)
 
-    assert s11 == "Unknown"
-    assert s22 == "Unknown"
-    assert s33 == "Unknown"
-    assert s44 == "Unknown"
+    assert s11 == ["Unknown", "Waht?"]
+    assert s22 == ["Unknown", "Waht?"]
+    assert s33 == ["Unknown", "Waht?"]
+    assert s44 == ["Unknown", "Waht?"]
   end
 
   test "Sparse Data Set Test: Micro examples should return correct classification" do
@@ -82,15 +87,15 @@ class SparseBayesTest < MicroTest::Test
     s33 = @@cls.classify(s3)
     s44 = @@cls.classify(s4)
 
-    assert s11 == "Refund"
-    assert s22 == "Partrefund"
-    assert s33 == "Nonrefund"
-    assert s44 == "Unknown"
+    assert s11 == ["Refund", "We are pleased to offer you a refund"]
+    assert s22 == ["Partrefund","You may receive a partial refund"]
+    assert s33 == ["Nonrefund", "Much apologies, no refund to you"]
+    assert s44 == ["Unknown", "Waht?"]
 
-    assert s11 != "Partrefund"
-    assert s22 != "Refund"
-    assert s33 != "Unknown"
-    assert s44 != "Nonrefund"
+    assert s11 != ["Partrefund", "You may receive a partial refund"]
+    assert s22 != ["Refund", "We are pleased to offer you a refund"]
+    assert s33 != ["Unknown", "Waht?"]
+    assert s44 != ["Nonrefund", "Much apologies, no refund to you"]
   end
 
 
@@ -106,18 +111,18 @@ test "Sparse Data Set Test: Micro examples should NOT match fake classes" do
     s33 = @@cls.classify(s3)
     s44 = @@cls.classify(s4)
 
-    assert s11 != "Computers"
-    assert s22 != "Science"
-    assert s33 != "Entertainment"
-    assert s44 != "Sports"
+    assert s11 != ["Computers", "computers yay!"]
+    assert s22 != ["Science", "science yay!"]
+    assert s33 != ["Entertainment", "entertainment yay!"]
+    assert s44 != ["Sports", "sports yay!"]
   end
 
 test "Sparse Data Set Test: Category counts are equivalent with number of training data per class" do
 
-    assert @@cls.category_counts[:Refund] == @@refund.count 
-    assert @@cls.category_counts[:Partrefund] == @@partrefund.count 
-    assert @@cls.category_counts[:Nonrefund] == @@norefund.count 
-    assert @@cls.category_counts[:Unknown] == @@unknown.count 
+    assert @@cls.category_counts[:Refund] == @@refund.count
+    assert @@cls.category_counts[:Partrefund] == @@partrefund.count
+    assert @@cls.category_counts[:Nonrefund] == @@norefund.count
+    assert @@cls.category_counts[:Unknown] == @@unknown.count
 
 end
 

data/test/trained_bayes_test.rb

@@ -8,7 +8,12 @@ class TrainedBayesTest < MicroTest::Test
   @@norefund = TeRex::Train::NONREFUND
   @@unknown = TeRex::Train::UNKNOWN
 
-  @@cls = TeRex::Classifier::Bayes.new("Refund", "Partrefund", "Nonrefund", "Unknown")
+  @@cls = TeRex::Classifier::Bayes.new(
+    {:tag => "Refund",     :msg => "We are pleased to offer you a refund"},
+    {:tag => "Partrefund", :msg => "You may receive a partial refund"},
+    {:tag => "Nonrefund",  :msg => "Much apologies, no refund to you"},
+    {:tag => "Unknown",    :msg => "Waht?"}
+  )
   @@refund.each {|txt| @@cls.train("Refund", txt) }
   @@partrefund.each {|txt| @@cls.train("Partrefund", txt) }
   @@norefund.each {|txt| @@cls.train("Nonrefund", txt) }
@@ -27,18 +32,18 @@ class TrainedBayesTest < MicroTest::Test
     s_unk1= @@cls.classify(s_unk)
 
     # We are lenient on Partrefund || Refund but we still want to see when it fails
-    assert s_refund1 == "Refund" || "Partrefund"
+    assert s_refund1 == ["Refund", "We are pleased to offer you a refund"] || ["Partrefund", "You may receive a partial refund"]
     # We are lenient on Refund || Partrefund because of the non-distinctness of the two.
-    assert s_partial1 == "Partrefund" || "Refund"
-    assert s_non1 == "Nonrefund"
-    assert s_unk1 == "Unknown"
+    assert s_partial1 == ["Partrefund", "You may receive a partial refund"] || ["Refund", "We are pleased to offer you a refund"]
+    assert s_non1 == ["Nonrefund", "Much apologies, no refund to you"]
+    assert s_unk1 == ["Unknown", "Waht?"]
 
     # We are lenient on Partrefund || Refund but we still want to see when it fails
-    assert s_refund1 != "Partrefund"
+    assert s_refund1 != ["Partrefund", "You may receive a partial refund"]
     # We are lenient on Refund || Partrefund but we still want to see when it fails
-    assert s_partial1 != "Refund"
-    assert s_non1 != "Unknown"
-    assert s_unk1 != "Nonrefund"
+    assert s_partial1 != ["Refund", "We are pleased to offer you a refund"]
+    assert s_non1 != ["Unknown", "Waht?"]
+    assert s_unk1 != ["Nonrefund", "Much apologies, no refund to you"]
   end
 
 
@@ -54,10 +59,10 @@ class TrainedBayesTest < MicroTest::Test
     norefund_s11 = @@cls.classify(norefund_s1)
     unk_s11 = @@cls.classify(unk_s1)
 
-    assert refund_s11 == "Refund"
-    assert partrefund_s11 == "Partrefund"
-    assert norefund_s11 == "Nonrefund"
-    assert unk_s11 == "Unknown"
+    assert refund_s11 == ["Refund", "We are pleased to offer you a refund"]
+    assert partrefund_s11 == ["Partrefund", "You may receive a partial refund"]
+    assert norefund_s11 == ["Nonrefund", "Much apologies, no refund to you"]
+    assert unk_s11 == ["Unknown", "Waht?"]
   end
 
   test "Training Data Set Test: Micro examples should return correct classification" do
@@ -72,15 +77,15 @@ class TrainedBayesTest < MicroTest::Test
     s33 = @@cls.classify(s3)
     s44 = @@cls.classify(s4)
 
-    assert s11 == "Refund"
-    assert s22 == "Partrefund"
-    assert s33 == "Nonrefund"
-    assert s44 == "Unknown"
+    assert s11 == ["Refund", "We are pleased to offer you a refund"]
+    assert s22 == ["Partrefund", "You may receive a partial refund"]
+    assert s33 == ["Nonrefund", "Much apologies, no refund to you"]
+    assert s44 == ["Unknown", "Waht?"]
 
-    assert s11 != "Partrefund"
-    assert s22 != "Refund"
-    assert s33 != "Unknown"
-    assert s44 != "Nonrefund"
+    assert s11 != ["Partrefund", "You may receive a partial refund"]
+    assert s22 != ["Nonrefund", "Much apologies, no refund to you"]
+    assert s33 != ["Unknown", "Waht?"]
+    assert s44 != ["Refund", "We are pleased to offer you a refund"]
   end
 
   test "Training Data Set Test: Micro examples should NOT match fake classes" do
@@ -95,10 +100,10 @@ class TrainedBayesTest < MicroTest::Test
     s33 = @@cls.classify(s3)
     s44 = @@cls.classify(s4)
 
-    assert s11 != "Computers"
-    assert s22 != "Science"
-    assert s33 != "Entertainment"
-    assert s44 != "Sports"
+    assert s11 != ["Computers", "computers yay!"]
+    assert s22 != ["Science", "science yay!"]
+    assert s33 != ["Entertainment", "entertainment yay!"]
+    assert s44 != ["Sports", "sports yay!"]
   end
 
   test "Training Data Set Test: Ambiguous examples should return 'Unknown'" do
@@ -113,10 +118,10 @@ class TrainedBayesTest < MicroTest::Test
     s33 = @@cls.classify(s3)
     s44 = @@cls.classify(s4)
 
-    assert s11 == "Unknown"
-    assert s22 == "Unknown"
-    assert s33 == "Unknown"
-    assert s44 == "Unknown"
+    assert s11 == ["Unknown", "Waht?"]
+    assert s22 == ["Unknown", "Waht?"] 
+    assert s33 == ["Unknown", "Waht?"]
+    assert s44 == ["Unknown", "Waht?"]
   end
 
   test "Training Data Set Test: Category counts are equivalent with number of training data per class" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: te_rex
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.11
 platform: ruby
 authors:
 - Joshua Bowles
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-06 00:00:00.000000000 Z
+date: 2014-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fast-stemmer
@@ -150,20 +150,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.4.9.1
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Simple text processing for small data sets.
 email:
 - jbowayles@gmail.com
@@ -171,8 +157,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/format/corpus/basic_file.rb
-- lib/format/corpus/brown_file.rb
+- lib/format/basic_file.rb
+- lib/format/brown_file.rb
+- lib/format/error_file.rb
 - lib/format/format.rb
 - lib/te_rex.rb
 - lib/te_rex/alpha_num.rb
@@ -183,8 +170,8 @@ files:
 - lib/te_rex/version.rb
 - test/alpha_num_test.rb
 - test/bayes_data_test.rb
+- test/corpus_test.rb
 - test/sparse_bayes_test.rb
-- test/test_helper.rb
 - test/test_modules/nonrefund.rb
 - test/test_modules/partrefund.rb
 - test/test_modules/refund.rb
@@ -210,7 +197,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: Basic NLP stuff for small data sets. Naive Bayes classification and corpora
@@ -218,8 +205,8 @@ summary: Basic NLP stuff for small data sets. Naive Bayes classification and cor
 test_files:
 - test/alpha_num_test.rb
 - test/bayes_data_test.rb
+- test/corpus_test.rb
 - test/sparse_bayes_test.rb
-- test/test_helper.rb
 - test/test_modules/nonrefund.rb
 - test/test_modules/partrefund.rb
 - test/test_modules/refund.rb

data/test/test_helper.rb

@@ -1,9 +0,0 @@
-require 'simplecov'
-SimpleCov.command_name 'mt'
-
-SimpleCov.start  do
-  add_filter 'test/modules'
-
-  add_group 'Formatter', 'lib/format'
-  add_group 'Core', 'lib/te_lrex'
-end