te_rex 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0a89389faff81b10c963d6469dfb52e514387813
+  data.tar.gz: 443a437ac01c887945eb14af4cc36aee5257977e
+SHA512:
+  metadata.gz: a03cd6e851150d5d9fdfced7942b1b7d9e0d20f47c08d5ac81edbd10337568c148b072a41f40b5615c2610d52535a1c6045874cac4c3baee90c18abe58143647
+  data.tar.gz: 2efe3da36bb2981b519ef4d5310915f7cb0906f1481a27dca3ab041a2847de0354feca7ec60f9404a5f6837ae0561b4f41b7e3454d892971ba39a64cc0030b86

data/lib/format/corpus/basic_file.rb

@@ -0,0 +1,29 @@
+module TeRex
+  module Format
+    class BasicFile
+
+      attr_accessor :sentences
+
+      def initialize(file_path)
+        @path = file_path
+      end
+
+      # Each line of file with Array object,
+      # strip it, split by whitespace, map it,
+      # split words by '/' to separate POS tags,
+      # join by whitespace
+      def scanner
+        @sentences ||= File.open(@path) do |file|
+          file.each_line.each_with_object([]) do |line, acc|
+            stripped_line = line.strip
+
+            unless stripped_line.nil? || stripped_line.empty?
+              acc << line.split(' ').map {|word| word}.join(' ')
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/format/corpus/brown_file.rb

@@ -0,0 +1,31 @@
+module TeRex
+  module Format
+    class BrownFile
+
+      attr_accessor :sentences
+
+      def initialize(file_path)
+        @path = file_path
+      end
+
+      # Each line of file with Array object,
+      # strip it, split by whitespace, map it,
+      # split words by '/' to separate POS tags,
+      # join by whitespace
+      def scanner
+        @sentences ||= File.open(@path) do |file|
+          file.each_line.each_with_object([]) do |line, acc|
+            stripped_line = line.strip
+
+            unless stripped_line.nil? || stripped_line.empty?
+              acc << line.split(' ').map do |word|
+                word.split('/').first
+              end.join(' ')
+            end
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/format/format.rb

@@ -0,0 +1,7 @@
+module TeRex
+  module Format
+    def self.category_term(t)
+      t.capitalize.intern
+    end
+  end
+end

data/lib/te_rex/alpha_num.rb

@@ -0,0 +1,36 @@
+# This module generates an alphabet of alpha-numeric characters for use in generating fake data.
+module TeRex
+  module AlphaNum
+    @symbols = ['!','@','#','$','%','^','&','*','(',')','-','+','=','<','>','?','~']
+    def self.gen
+      ('a'..'z').map{|aleph| aleph}
+      .concat(('A'..'Z').map{|aleph2| aleph2})
+      .concat(@symbols)
+      .concat((1..20).map{|num| num})
+    end
+
+    #Some randomized date format generator.
+    def self.date
+      day = (1..31).map{|num| num}.sample
+      month = (1..12).map{|num| num}.sample
+      year2 = (10..99).map{|num| num}.sample
+      year4 = (1900..2099).map{|n| n}.sample
+      ["#{day}-#{month}-#{year2}",
+      "#{day}/#{month}/#{year2}",
+      "#{day}/#{month}/#{year4}",
+      "#{day}-#{month}-#{year4}",
+      "#{year4}-#{month}-#{day}",
+      "#{year4}/#{month}/#{day}",
+      "#{year2}/#{month}/#{day}",
+      "#{year2}-#{month}-#{day}",
+      "#{month}-#{day}-#{year2}",
+      "#{month}/#{day}/#{year2}",
+      "#{month}/#{day}/#{year4}",
+      "#{month}-#{day}-#{year4}",
+      "#{year4}-#{day}-#{month}",
+      "#{year4}/#{day}/#{month}",
+      "#{year2}/#{day}/#{month}",
+      "#{year2}-#{day}-#{month}"].sample
+    end
+  end
+end

data/lib/te_rex/bayes.rb

@@ -0,0 +1,73 @@
+#
+# Refactor of Lucas Carlson's classifier https://github.com/cardmagic/classifier (Copyright (c) 2005 lucas@rufy.com)
+#
+module TeRex
+  module Classifier
+    class Bayes
+
+      attr_accessor :category_counts, :total_words
+
+      def initialize(*categories)
+        @clasif = Hash.new 
+        categories.each {|cat| @clasif[TeRex::Format.category_term(cat)] = Hash.new}
+        @total_words = 0
+        @category_counts = Hash.new(0)
+      end
+
+      def train(ctgry, text)
+        category = TeRex::Format.category_term(ctgry)
+        @category_counts[category] += 1
+
+        BayesData.index_frequency(text).each do |word, count| 
+          @clasif[category][word] ||= 0
+          @clasif[category][word] += count
+
+          @total_words += count
+        end
+      end
+
+      def classifications(text)
+        score = Hash.new
+        training_count = @category_counts.values.inject {|x,y| x+y}.to_f
+
+        @clasif.each do |category, category_words|
+          score[category.to_s] = 0
+          total = category_words.values.inject(0) {|sum, element| sum+element}
+          BayesData.index_frequency(text).each do |word, count|
+            s = category_words.has_key?(word) ? category_words[word] : 0.1
+            score[category.to_s] += Math.log(s/total.to_f)
+          end
+
+          k = @category_counts.has_key?(category) ? @category_counts[category] : 0.1
+          score[category.to_s] += Math.log(k/training_count)
+        end
+
+        score
+      end
+
+      def classify(text)
+        (classifications(text).sort_by{|a| -a[1]})[0][0]
+      end
+
+      def categories
+        @classif.keys.collect {|c| c.to_s}
+      end
+
+      def training_description
+        max_threshold = (@total_words/self.category_counts.keys.count).to_f
+        tmp = []
+        @clasif.each_pair do |term,val|
+          cc = self.category_counts[term]
+          train_ratio = (@total_words/cc).to_f
+          tmp << [(train_ratio > max_threshold), term, "description" => {"training_ratio" => "#{train_ratio}", "threshold" => "#{max_threshold}", "category_counts" => "#{cc}", "total_words" => "#{@total_words}"}]
+        end
+        tmp
+      end
+
+      def under_trained?
+        training_description.select {|ut| ut.first == true}
+      end
+
+    end
+  end
+end

data/lib/te_rex/bayes_data.rb

@@ -0,0 +1,100 @@
+require 'fast_stemmer'
+
+module TeRex
+  module Classifier
+    class BayesData
+      class << self
+
+        # Remove all kinds of explicit punctuation. 
+        def remove_punct(s)
+          s.gsub(/(\,)|(\?)|(\.)|(\!)|(\;)|(\:)|(\")|(\@)|(\#)|(\$)|(\^)|(\&)|(\*)|(\()|(\))|(\_)|(\=)|(\+)|(\[)|(\])|(\\)|(\|)|(\<)|(\>)|(\/)|(\`)|(\{)|(\})/, '')
+        end
+
+        # Remove cardinal terms (1st, 23rd, 42nd)
+        def remove_cardinal(s)
+          s.gsub(/\d+\w{2}/, '')
+        end
+
+        # Replace date times with TERM (09MAR04, 02-23-14, 2014/03/05)
+        def date_time(s)
+          s.gsub(/(^\d+)|(\s\d+(AM|PM))|(\d{2}\w{3}\d{2})|(\d{2}\:\d{2})|(\d{2,4}\-\d{2,4}-\d{2,4})|(\d{1,3}\/\d{2,4}\/\d{2,4})|(\d+\:\d+)/, 'datetime')
+        end
+
+        # Replace money types with TERM ($60, 120.00, $423.89)
+        def money_term(s)
+          s.gsub(/(\$\d+\.\d+)|(\$\d+)|(\d+\.\d+)/, 'moneyterm')
+        end
+
+        # Return a Hashed Index of words => instance_count. 
+        # Each word in the string is interned and shows count in the document.
+        def index_frequency(text)
+          cfi = clean_stemmed_filtered_index(text)
+          cni = clean_filtered_index(text)
+          cfi.merge(cni)
+        end
+
+        # Return text with datetime and moneyterms replaced, remove cardinal terms (1st, 23rd, 42nd), remove punctuation.
+        # At one point we were replacing any non-word chars exlcuding spaces (/[^\w\s]/) like so `gsub(/[^\w\s]/, "")` but I took it out as it removed some punctuation needed to distinguish some classes.
+        def clean(text)
+          dt = date_time(text)
+          mt = money_term(dt)
+          rp = remove_punct(mt)
+          remove_cardinal(rp)
+        end
+
+        # Return a filtered word freq index with stemmed morphemes and without extra punctuation or short words
+        def clean_stemmed_filtered_index(text)
+          stemmed_filtered_index clean(text).split
+        end
+
+        # Return a filtered word freq index without extra punctuation or short words
+        def clean_filtered_index(text)
+          filtered_index clean(text).split
+        end
+
+        # Return a word freq index without downcasing, stemming, or filtering with stop list
+        def clean_naive_index(text)
+          naive_index clean(text).split
+        end
+
+        private
+        # Downcase, filter against stop list, ignore sequences less that 2 chars, and stem words
+        def stemmed_filtered_index(word_array)
+          idx = Hash.new(0)
+          word_array.each do |word|
+            word.downcase!
+            if !TeRex::StopWord::LIST.include?(word) #&& word.length > 2
+              idx[word.stem.intern] += 1
+            end
+          end
+
+          idx
+        end
+
+        # Downcase, filter against stop list, and ignore sequences less that 2 chars.
+        def filtered_index(word_array)
+          idx = Hash.new(0)
+          word_array.each do |word|
+            word.downcase!
+            if !TeRex::StopWord::LIST.include?(word) #&& word.length > 2
+              idx[word.intern] += 1
+            end
+          end
+
+          idx
+        end
+
+        # Count everything in the word array.
+        def naive_index(word_array)
+          idx = Hash.new(0)
+          word_array.each do |word|
+            idx[word.intern] += 1
+          end
+
+          idx
+        end
+
+      end
+    end
+  end
+end

data/lib/te_rex/corpus.rb

@@ -0,0 +1,42 @@
+module TeRex
+  module Corpus
+    class Body
+
+      attr_accessor :files, :sample_size, :training, :testing
+
+      def initialize(glob, klass)
+        @glob = glob
+        @klass = klass
+      end
+
+      def build
+        get_files
+        @training_sentences = partition_train
+        @testing_sentences = partition_test
+      end
+
+      def get_files
+        @files ||= Dir[@glob].map do |file|
+          @klass.new(file)
+        end
+        @sample_size = (@files.count * 0.75).round
+        @files
+      end
+
+      def partition_train
+        @files[0..@sample_size].map do |file|
+          file.scanner
+        end.flatten
+      end
+
+      def partition_test
+        @files[(@sample_size - 1)..-1].map do |file|
+          file.scanner
+        end.flatten
+      end
+
+    end
+  end
+end
+
+

data/lib/te_rex/stop_word.rb

@@ -0,0 +1,68 @@
+module TeRex
+  class StopWord
+    LIST = [
+      "a",
+      "am",
+      "an",
+      "and",
+      "are",
+      "as",
+      "at",
+      "be",
+      "by",
+      "do",
+      "for",
+      "in",
+      "into",
+      "it",
+      "it's",
+      "its",
+      "of",
+      "so",
+      "than",
+      "that",
+      "that's",
+      "the",
+      "what",
+      "what's",
+      "where",
+      "which",
+      "january",
+      "february",
+      "march",
+      "april",
+      "may",
+      "june",
+      "july",
+      "august",
+      "september",
+      "october",
+      "november",
+      "december",
+      "jan",
+      "feb",
+      "mar",
+      "apr",
+      "aug",
+      "sept",
+      "nov",
+      "dec",
+      "monday",
+      "mon",
+      "tuesday",
+      "tue",
+      "wednesday",
+      "wed",
+      "thursday",
+      "thur",
+      "friday",
+      "fri",
+      "saturday",
+      "sat",
+      "sunday",
+      "sun",
+      "pm",
+      "am"
+    ]
+  end
+end

data/lib/te_rex/version.rb

@@ -0,0 +1,3 @@
+module TeRex
+  VERSION = "0.0.4"
+end

data/lib/te_rex.rb

@@ -0,0 +1,11 @@
+require_relative "format/format"
+require_relative "format/corpus/brown_file"
+require_relative "format/corpus/basic_file"
+require_relative "te_rex/stop_word"
+require_relative "te_rex/alpha_num"
+require_relative "te_rex/bayes_data"
+require_relative "te_rex/bayes"
+require_relative "te_rex/corpus"
+
+module TeRex
+end

data/test/alpha_num_test.rb

@@ -0,0 +1,18 @@
+require_relative "../lib/te_rex"
+class AlphaNumTest < MicroTest::Test
+
+  test "generates array of lowercase roman characters" do
+    res = TeRex::AlphaNum.gen
+    ('a'..'z').each {|char| assert res.include? char}
+  end
+
+  test "generates array of uppercase roman characters" do
+    res = TeRex::AlphaNum.gen
+    ('A'..'Z').each {|char| assert res.include? char}
+  end
+
+  test "generates array of integers 1-20" do
+    res = TeRex::AlphaNum.gen
+    (1..20).each {|int| assert res.include? int}
+  end
+end

data/test/bayes_data_test.rb

@@ -0,0 +1,70 @@
+require_relative "../lib/te_rex"
+class BayesDataTest < MicroTest::Test
+
+  test "punctuation is removed (except %)" do
+    s1 = "This * punctuation se%ntence ).!"
+    s2 = "Much $ in @ this } [ sentence too?"
+    s3 = "And I$ have c#des in |his one with 100% refund too@>."
+
+    s11 = TeRex::Classifier::BayesData.remove_punct(s1)
+    s22 = TeRex::Classifier::BayesData.remove_punct(s2)
+    s33 = TeRex::Classifier::BayesData.remove_punct(s3)
+
+    assert s11 == "This  punctuation se%ntence "
+    assert s22 == "Much  in  this   sentence too"
+    assert s33 == "And I have cdes in his one with 100% refund too"
+  end
+
+  test "datetime is removed and replaced" do
+    s1 = "This $140 will be paid on 09/14/2014"
+    s2 = "I get $20.00 on 2014-05-21 and on 09MAR04"
+    s3 = "I'll pay you $60.21 on 06-20-2014"
+
+    s11 = TeRex::Classifier::BayesData.date_time(s1)
+    s22 = TeRex::Classifier::BayesData.date_time(s2)
+    s33 = TeRex::Classifier::BayesData.date_time(s3)
+
+    assert s11 == "This $140 will be paid on datetime"
+    assert s22 == "I get $20.00 on datetime and on datetime"
+    assert s33 == "I'll pay you $60.21 on datetime"
+  end
+
+  test "moneyterm is removed and replaced" do
+    s1 = "$140 will be paid on 09/14/2014 with $60"
+    s2 = "I get $20.00 on 2014-05-21 and on 09MAR04"
+    s3 = "You'll make $1234.73 on 06-20-2014"
+
+    s11 = TeRex::Classifier::BayesData.money_term(s1)
+    s22 = TeRex::Classifier::BayesData.money_term(s2)
+    s33 = TeRex::Classifier::BayesData.money_term(s3)
+
+    assert s11 == "moneyterm will be paid on 09/14/2014 with moneyterm"
+    assert s22 == "I get moneyterm on 2014-05-21 and on 09MAR04"
+    assert s33 == "You'll make moneyterm on 06-20-2014"
+  end
+
+  test "cleaned text does what we want" do
+    s1 = "$140 will be paid on 09/14/2014 with $60"
+    s2 = "I get $20.00 on 2014-05-21 and on 09MAR04 with %49 and &*%^)"
+    s3 = "And I$ have c#des in |his one wi%th 100% refund too@>."
+
+    s11 = TeRex::Classifier::BayesData.clean(s1)
+    s22 = TeRex::Classifier::BayesData.clean(s2)
+    s33 = TeRex::Classifier::BayesData.clean(s3)
+
+    assert s11 == "moneyterm will be paid on datetime with moneyterm"
+    assert s22 == "I get moneyterm on datetime and on datetime with %49 and %"
+    assert s33 == "And I have cdes in his one wi%th % refund too"
+  end
+
+  test "index frequency has correct counts" do
+    s = "Here is a sentence $141.34 that that $60 that 123.56 I need & & ^ % $c#@ to check the index is correct and okay."
+    result = TeRex::Classifier::BayesData.index_frequency(s)
+
+    assert result[:moneyterm] == 3
+    assert result[:sentenc] == 1
+    assert result[:sentence] == 1
+
+  end
+
+end

data/test/sparse_bayes_test.rb

@@ -0,0 +1,130 @@
+require_relative "../lib/te_rex"
+class SparseBayesTest < MicroTest::Test
+  @@refund = [
+    "Free cancellation before 1201 AM on 9/17/14! If you cancel or change your reservation after 1201 AM on 9/17/14 the hotel will charge you for the total cost of your reservation.",
+    "ALL RESERVATIONS MUST BE CANCELLED 24 HOURS PRIOR TO HOST TIME UNLESS DEPOSIT REQUIRED IF THIS RESERVATION HAS BEEN MADE ELECTRONICALLY PLEASE CANCEL IT ELECTRONICALLY TO AVOID CONFUSION AND A NO SHOW BILL. POLICY SUBJECT TO CHANGE. .",
+    "Free cancellation before 800 PM on 9/20/14! If you cancel or change your reservation after 800 PM on 9/20/14 the hotel will charge you $158. If you cancel or change your reservation after 800 PM on 9/21/14 the hotel will charge you for the total cost of your reservation."
+  ]
+
+  @@partrefund = [
+    "If you cancel or change your reservation before 1201 AM on 10/21/14 the hotel will charge you $57. If you cancel or change your reservation after 1201 AM on 10/21/14 the hotel will charge you $335. If you cancel or change your reservation after 1201 AM on 10/24/14 the hotel will charge you for the total cost of your reservation.",
+    "If you cancel or change your reservation before 1201 AM on 9/10/14 the hotel will charge you $225. If you cancel or change your reservation after 1201 AM on 9/10/14 the hotel will charge you for the total cost of your reservation.",
+    "Cancellations or changes made before 4:00 PM Eastern Time on Sep 11, 2014 are subject to a 1 Night Room & Tax penalty. Cancellations or changes made after 4:00 PM Eastern Time on Sep 11, 2014 are subject to a 1 Night Room & Tax penalty. The property makes no refunds for no shows or early checkouts."
+  ]
+
+  @@norefund = [
+    "This reservation is non-refundable. Cancellations or changes made at any time are subject to a 100% charge.",
+    "This rate is non-refundable and cannot be changed or cancelled - if you do choose to change or cancel this booking you will not be refunded any of the payment.",
+    "For the room type and rate that you've selected you are not allowed to change or cancel your reservation. If you cancel your room you will still be charged for the full reservation amount."
+  ]
+
+  @@unknown = [
+    "The cancellation policy will be determined when the rate is validated."
+  ]
+
+  @@cls = TeRex::Classifier::Bayes.new("Refund", "Partrefund", "Nonrefund", "Unknown")
+  @@refund.each {|txt| @@cls.train("Refund", txt) }
+  @@partrefund.each {|txt| @@cls.train("Partrefund", txt) }
+  @@norefund.each {|txt| @@cls.train("Nonrefund", txt) }
+  @@unknown.each {|txt| @@cls.train("Unknown", txt) }
+
+  test "Sparse Data Set Test: Random exact match sould classify correctly" do
+
+    s_refund = @@refund.sample
+    s_partial = @@partrefund.sample
+    s_non = @@norefund.sample
+    s_unk = @@unknown.sample
+
+    s_refund1 = @@cls.classify(s_refund)
+    s_partial1 = @@cls.classify(s_partial)
+    s_non1 = @@cls.classify(s_non)
+    s_unk1= @@cls.classify(s_unk)
+
+    assert s_refund1 == "Refund"
+    assert s_partial1 == "Partrefund"
+    assert s_non1 == "Nonrefund"
+    assert s_unk1 == "Unknown"
+
+    assert s_refund1 != "Partrefund"
+    assert s_partial1 != "Refund"
+    assert s_non1 != "Unknown"
+    assert s_unk1 != "Nonrefund"
+  end
+
+
+  test "Sparse Data Set Test: Non-canonical examples should return unknown" do
+
+    s1 = "You will get a full refund and free cancellation"
+    s2 = "You will get a partial refund and be charged"
+    s3 = "You will get non refund"
+    s4 = "You will get a nonsense am I writing here."
+
+    s11 = @@cls.classify(s1)
+    s22 = @@cls.classify(s2)
+    s33 = @@cls.classify(s3)
+    s44 = @@cls.classify(s4)
+
+    assert s11 == "Unknown"
+    assert s22 == "Unknown"
+    assert s33 == "Unknown"
+    assert s44 == "Unknown"
+  end
+
+  test "Sparse Data Set Test: Micro examples should return correct classification" do
+
+    s1 = "Free cancellation before"
+    s2 = "If you cancel or change your reservation before"
+    s3 = "non-refund"
+    s4 = "policy rate validated."
+
+    s11 = @@cls.classify(s1)
+    s22 = @@cls.classify(s2)
+    s33 = @@cls.classify(s3)
+    s44 = @@cls.classify(s4)
+
+    assert s11 == "Refund"
+    assert s22 == "Partrefund"
+    assert s33 == "Nonrefund"
+    assert s44 == "Unknown"
+
+    assert s11 != "Partrefund"
+    assert s22 != "Refund"
+    assert s33 != "Unknown"
+    assert s44 != "Nonrefund"
+  end
+
+
+test "Sparse Data Set Test: Micro examples should NOT match fake classes" do
+
+    s1 = "free cancellation"
+    s2 = "partial refund"
+    s3 = "no refund"
+    s4 = "policy rate validated."
+
+    s11 = @@cls.classify(s1)
+    s22 = @@cls.classify(s2)
+    s33 = @@cls.classify(s3)
+    s44 = @@cls.classify(s4)
+
+    assert s11 != "Computers"
+    assert s22 != "Science"
+    assert s33 != "Entertainment"
+    assert s44 != "Sports"
+  end
+
+test "Sparse Data Set Test: Category counts are equivalent with number of training data per class" do
+
+    assert @@cls.category_counts[:Refund] == @@refund.count 
+    assert @@cls.category_counts[:Partrefund] == @@partrefund.count 
+    assert @@cls.category_counts[:Nonrefund] == @@norefund.count 
+    assert @@cls.category_counts[:Unknown] == @@unknown.count 
+
+end
+
+test "Sparse Data Set Test: All SPARSE Training classes should be undertrained... " do
+  res = @@cls.under_trained?
+  assert res.count == 4
+end
+
+end
+

data/test/test_modules/nonrefund.rb

@@ -0,0 +1,35 @@
+module TeRex
+  module Train
+    NONREFUND = [
+      "This reservation is non-refundable. Cancellations or changes made at any time are subject to a 100% charge.",
+      "This reservation is non-refundable. Cancellations made at any time are subject to a 100% charge.",
+      "This reservation is non-refundable.",
+      "This reservation is non-refundable. Cancellations or changes made at any time are subject to a 100% charge.",
+      "This rate is non-refundable and cannot be changed or cancelled - if you do choose to change or cancel this booking you will not be refunded any of the payment.",
+      "This reservation is non-refundable. Cancellations made at any time are subject to a 100% charge.",
+      "This reservation is non-refundable.",
+      "This reservation is non-refundable. Cancellations or changes made at any time are subject to a 100% charge.",
+      "This reservation is non-refundable. Cancellations made at any time are subject to a 100% charge.",
+      "This reservation is non-refundable.",
+      "This rate is non-refundable and cannot be changed or cancelled - if you choose to change or cancel this booking you will not be refunded any of the payment.",
+      "This rate is non-refundable and cannot be changed or cancelled - if you do choose to cancel this booking you will not be refunded any of the payment.",
+      "This rate is non-refundable and cannot be changed or cancelled - if you do choose to change or cancel this booking you will not be refunded any payment.",
+      "This rate is non-refundable and cannot be changed or cancelled - if you do choose to change or cancel this booking you will not be refunded any of the payment.",
+      "This rate is non-refundable and cannot be changed or cancelled - if you do choose to change or cancel this booking you will not be refunded any of the payment.",
+      "This rate is non-refundable and cannot be changed or cancelled - if you do choose to change or cancel this booking you will not be refunded any of the payment.",
+      "For the room type and rate that you've selected you are not allowed to change or cancel your reservation. If you cancel your room you will still be charged for the full reservation amount.",
+      "For the room type and rate that you've selected you are not allowed to change or cancel your reservation. If you cancel your room you will still be charged for the full reservation amount.",
+      "For the room type and rate that you've selected you are not allowed to change or cancel your reservation. If you cancel your room you will still be charged for the full reservation amount.",
+      "This rate is non-refundable and cannot be changed or cancelled - if you do choose to change or cancel this booking you will not be refunded any of the payment.",
+      "For the room type and rate that you've selected you are not allowed to change or cancel your reservation. If you cancel your room you will still be charged for the full reservation amount.",
+      "For the room type and rate that you've selected you are not allowed to change or cancel your reservation. If you cancel your room you will still be charged for the full reservation amount.",
+      "For the room type and rate that you've selected you are not allowed to change or cancel your reservation. If you cancel your room you will still be charged for the full reservation amount.",
+      "This reservation is non-refundable. Cancellations or changes made at any time are subject to a 100% charge. Refunds are not available for early check-out.",
+      "Non-refundable. Cancellations or changes made at any time are subject to a 100% charge. We are sorry but refunds are not available for early check-out.",
+      "This reservation is non-refundable. Cancellations or changes made at any time are subject to a 100% charge. We are sorry but refunds are not available for early check-out.",
+      "This reservation is non-refundable. Cancellations or changes made at any time are subject to a 100% charge. We are sorry but refunds are not available for early check-out.",
+      "This reservation is non-refundable. Cancellations or changes made at any time are subject to a 100% charge. We are sorry but refunds are not available for early check-out.",
+      "This reservation is non-refundable. Cancellations or changes made at any time are subject to a 100% charge. We are sorry but refunds are not available for early check-out."
+    ]
+  end
+end

data/test/test_modules/partrefund.rb

@@ -0,0 +1,30 @@
+module TeRex
+  module Train
+    PARTREFUND = [
+      "If you cancel or change your reservation before 1201 AM on 10/21/14 the hotel will charge you $57. If you cancel or change your reservation after 1201 AM on 10/21/14 the hotel will charge you $335. If you cancel or change your reservation after 1201 AM on 10/24/14 the hotel will charge you for the total cost of your reservation.",
+      "If you cancel or change your reservation before 4:00 PM on 9/22/14, the hotel will charge you $388. If you cancel or change your reservation after 4:00 PM on 9/22/14, the hotel will charge you for the total cost of your reservation. ",
+      "If you cancel or change your reservation before 1201 AM on 9/10/14 the hotel will charge you $225. If you cancel or change your reservation after 1201 AM on 9/10/14 the hotel will charge you for the total cost of your reservation.",
+      "Cancellations or changes made before 4:00 PM Eastern Time on Sep 11, 2014 are subject to a 1 Night Room & Tax penalty. Cancellations or changes made after 4:00 PM Eastern Time on Sep 11, 2014 are subject to a 1 Night Room & Tax penalty. The property makes no refunds for no shows or early checkouts.",
+     "If you cancel or change your reservation before 3:00 PM on 07/14/14, the hotel will charge you $189. If you cancel or change your reservation after 3:00 PM on 07/14/14, the hotel will charge you $225. If you cancel or change your reservation after 3:00 PM on 07/17/14 the hotel will charge you for the total cost of your reservation. ",
+      "If you cancel or change your reservation before 3:00 PM on 7\\/31\\/14, the hotel will charge you $305. If you cancel or change your reservation after 3:00 PM on 7\\/31\\/14, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 4:00 PM on 9/22/14, the hotel will charge you $245. If you cancel or change your reservation after 4:00 PM on 9/22/14, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 4:00 PM on 9/24/14, the hotel will charge you $233. If you cancel or change your reservation after 4:00 PM on 9/24/14, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 12:00 PM on 1/05/15, the hotel will charge you $86. If you cancel or change your reservation after 12:00 PM on 1/05/15, the hotel will charge you $625. If you cancel or change your reservation after 12:00 PM on 1/07/15 the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 4:00 PM on 9/21/14, the hotel will charge you $215. If you cancel or change your reservation after 4:00 PM on 9/21/14, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 6:00 PM on 04/17/14, the hotel will charge you $187. If you cancel or change your reservation after 6:00 PM on 04/17/14, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 16:00 PM on 3/31/14, the hotel will charge you $89. If you cancel or change your reservation after 16:00 PM on 3/31/14,, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 6:00 AM on 12/07/14, the hotel will charge you $5675. If you cancel or change your reservation after 6:00 AM on 12/07/14, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 3:00 PM on 07/14/14, the hotel will charge you $189. If you cancel or change your reservation after 3:00 PM on 07/14/14, the hotel will charge you $225. If you cancel or change your reservation after 3:00 PM on 07/17/14 the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 7:00 AM on 06/09/14, the hotel will charge you $509. If you cancel or change your reservation after 7:00 AM on 06/09/14, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 9:00 PM on 5/23/14, the hotel will charge you $1018. If you cancel or change your reservation after 9:00 PM on 5/23/14, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 10:00 PM on 5/31/14, the hotel will charge you $215. If you cancel or change your reservation after 10:00 PM on 5/31/14, the hotel will charge you $425. If you cancel or change your reservation after 10:00 PM on 6/03/14 the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 8:00 AM on 12/17/14, the hotel will charge you $15. If you cancel or change your reservation after 8:00 AM on 12/17/14, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 12:00 PM on 1/05/15, the hotel will charge you $86. If you cancel or change your reservation after 12:00 PM on 1/05/15, the hotel will charge you $625. If you cancel or change your reservation after 12:00 PM on 1/07/15 the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 2:00 PM on 11/14/14, the hotel will charge you $57. If you cancel or change your reservation after 2:00 PM on 11/14/14, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 2:00 PM on 11/14/14, the hotel will charge you $57. If you cancel or change your reservation after 2:00 PM on 11/14/14, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 12:00 PM on 1/05/15, the hotel will charge you $86. If you cancel or change your reservation after 12:00 PM on 1/05/15, the hotel will charge you $625. If you cancel or change your reservation after 12:00 PM on 1/07/15 the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 4:00 PM on 2/21/15, the hotel will charge you $115. If you cancel or change your reservation after 4:00 PM on 2/21/15, the hotel will charge you for the total cost of your reservation. ",
+     "If you cancel or change your reservation before 4:00 PM on 2/14/15, the hotel will charge you $318. If you cancel or change your reservation after 4:00 PM on 2/14/15, the hotel will charge you for the total cost of your reservation. ",
+    ]
+  end
+end

data/test/test_modules/refund.rb

@@ -0,0 +1,32 @@
+module TeRex
+  module Train
+    REFUND = [
+      "Free cancellation before 1201 AM on 9/17/14! If you cancel or change your reservation after 1201 AM on 9/17/14 the hotel will charge you for the total cost of your reservation. ",
+      "Free cancellation before 12:01 AM on 10/29/14! If you cancel or change your reservation after 12:01 AM on 10/29/14, the hotel will charge you $194. If you cancel or change your reservation after 12:01 AM on 10/30/14, the hotel will charge you for the total cost of your reservation.",
+      "If you cancel or change your reservation after 12:01 AM on 9/12/14, the hotel will charge you for the total cost of your reservation. ",
+      "Cancellations or changes made within 1 day prior to 4:00 PM local hotel time on the day of arrival are subject to a $142.93 charge. Cancellations or changes made after 4:00 PM local hotel time on the day of arrival are subject to a 100% charge. We are sorry but refunds are not available for early check-out.",
+      "We understand that sometimes your travel plans change. We do not charge a change or cancel fee. However, this property (Courtyard by Marriott Traverse City) imposes the following penalty to its customers that we are required to pass on: Cancellations or changes made after 5:00 PM ((GMT-05:00) Eastern Time (US & Canada)) on Jul 9, 2014 are subject to a 1 Night Room & Tax penalty. The property makes no refunds for no shows or early checkouts.",
+      "Free cancellation before 800 PM on 9/20/14! If you cancel or change your reservation after 800 PM on 9/20/14 the hotel will charge you $158. If you cancel or change your reservation after 800 PM on 9/21/14 the hotel will charge you for the total cost of your reservation. ",
+      "Any cancellation received within 2 days prior to arrival date will incur the first night charge. Failure to arrive at your hotel will be treated as a No-Show and will incur the first night charge (Hotel policy).",
+      "For the room type you've selected you can cancel your reservation for a full refund up until noon on Friday September 12th (local hotel time). If you decide to cancel your reservation anytime between noon on Friday September 12th and noon on Saturday September 13th (local hotel time) the hotel requires payment for the first night's stay. You will be charged for the first night's stay including taxes and fees. Any remaining amount will be refunded to you. Refunds or cancellations are not available after noon local hotel time on your day of arrival (Saturday September 13th). ",
+      "THIS PROPERTY REQUIRES A NOTIFICATION OF CANCELLATION BY 4PM HOTEL TIME 1 DAY PRIOR TO ARRIVAL TO AVOID A PENALTY.",
+      "You can cancel free of charge up until the cancellation window. Cancellations or changes made after 4:00 PM Eastern Time on Sep 12, 2014 are subject to a 1 Night Room & Tax penalty. The property makes no refunds for no shows or early checkouts.",
+      "For the room type you've selected you can cancel your reservation for a full refund up until noon on Monday September 15th (local hotel time). If you decide to cancel your reservation anytime between noon on Monday September 15th and noon on Wednesday September 17th (local hotel time) the hotel requires payment for the first night's stay. You will be charged for the first night's stay including taxes and fees. Any remaining amount will be refunded to you. Refunds or cancellations are not available after noon local hotel time on your day of arrival (Wednesday September 17th).",
+      "-08FEB04 - END - CANCEL BY 0 DAYS PRIOR TO ARRIVAL, LOCAL HOTEL TIME TO AVOID A CANCELLATION PENALTY CANCEL POLICIES AND EARLY DEPARTURE FEES VARY BY HOTEL. SINCE A HOTEL CAN SET A CANCELLATION POLICY OF UP TO 30 DAYS IN ADVANCE PLEASE REVIEW POLICY PRIOR TO BOOKING TO AVOID POSSIBLE CHARGE.",
+      "IF CANCELLATION IS NECESSARY, TO AVOID BILLING YOU MUST RECEIVE A CANCEL NUMBER BY 6 PM ON THE DATE OF ARRIVAL. CANCEL TIMES AND DATES MAY VARY. CONTACT THE RESERVED MOTEL 6 FOR SPECIFIC POLICIES THAT MAY AFFECT YOUR RESERVATION. . -EARLY CHECKOUT POLICY -CONTACT LOCATION FOR MORE INFORMATION. .",
+      "-1800 HOTEL TIME, DAY OF ARRIVAL TO AVOID BILLING OF 1 NIGHT ROOM AND TAX OR FORFEITURE OF DEPOSIT",
+      "RESERVATIONS MUST BE CANCELLED 1DAY/24HOURS PRIOR TO ARRIVAL TO AVOID A PENALTY OF ONE NIGHT ROOM AND TAX. INDIVIDUAL PLANS MAY HAVE VARING CANCEL POLICIES. EARLY DEPARTURE FEE 1NIGHTS STAY. .",
+      "06NOV03 - END - CANCEL BY 1 DAYS PRIOR TO 1600 HOURS ON DAY OF ARRIVAL, LOCAL HOTEL TIME TO AVOID A CANCELLATION PENALTY CANCELLATION POLICY TEXT - DUE TO SEASONAL VARIATIONS, MOST ACCURATE CANCEL POLICY RETURNED UPON BOOKINGS.  INTERNATIONAL   MAXIMUM 3 ROOMS PER RESERVATION.  MAXIMUM 3 PERSONS PER ROOM ONLY IF A ROLLAWAY IS AVAILABLE.  DOMESTIC   MAXIMUM 5 ROOMS PER RESERVATIONS.  MAXIMUM 5 PERSONS PER ROOM ONLY IF ROLLAWAY IS AVAILABLE.",
+      "RESERVATIONS MUST BE CANCELED *24 HOURS* PRIOR TO ARRIVAL DATE TO AVOID A PENALTY OF ONE NIGHT ROOM AND TAX CHARGE INDIVIDUAL PLANS MAY VARY SEE PLAN DISPLAY FOR MORE INFORMATION **100USD EARLY DEPARTURE FEE** .",
+      "ALL RESERVATIONS MUST BE CANCELLED 24 HOURS PRIOR TO HOST TIME UNLESS DEPOSIT REQUIRED IF THIS RESERVATION HAS BEEN MADE ELECTRONICALLY PLEASE CANCEL IT ELECTRONICALLY TO AVOID CONFUSION AND A NO SHOW BILL. POLICY SUBJECT TO CHANGE. .",
+      "ALL RESERVATIONS MUST BE CANCELLED 48 HOURS PRIOR TO ARRIVAL HOST TIME IF THIS RESERVATION HAS BEEN MADE ELECTRONICALLY PLEASE CANCEL IT ELECTRONICALLY TO AVOID CONFUSION AND A NO SHOW BILL POLICY SUBJECT TO CHANGE .",
+      "IF CANCELLATION IS NECESSARY, TO AVOID BILLING YOU MUST RECEIVE A CANCEL NUMBER BY 6 PM ON THE DATE OF ARRIVAL. CANCEL TIMES AND DATES MAY VARY. CONTACT THE RESERVED MOTEL 6 FOR SPECIFIC POLICIES THAT MAY AFFECT YOUR RESERVATION. . -EARLY CHECKOUT POLICY -CONTACT LOCATION FOR MORE INFORMATION. .o",
+      "RESERVATIONS MUST BE CANCELED *72 HOURS* PRIOR TO ARRIVAL LOCAL HOTEL TIME TO AVOID A PENALTY OF ONE NIGHT ROOM AND TAX CHARGE INDIVIDAUL PLANS MAY VARY SEE PLAN DISPLAY FOR MORE INFORMATION **75USD EARLY DEPARTURE FEE**",
+      "- 12 NOON HOTEL TIME DAY OF ARRIVAL TO AVOID BILLING OF 1NT ROOM AND TAX OR FORFEITURE OF DEPOSIT",
+      "RESERVATIONS MUST BE CANCELLED BY 1600/4PM DAY OF ARRIVAL LOCAL HOTEL TIME TO AVOID A PENALTY OF ONE NIGHT ROOM AND TAX CHARGE. NOTE- INDIVIDUAL PLANS MAY VARY. PLEASE READ INDIVIDUAL DISPLAY FOR MORE INFORMATION. **50.00USD EARLY DEPARTURE FEE** .",
+      "- RESERVATIONS BOOKED 8 DAYS OR MORE PRIOR TO ARRIVAL MUST BE CANCELLED 7 DAYS PRIOR TO ARRIVAL TO RECEIVE REFUND - RESERVATIONS BOOKED WITHIN 7 DAYS OF ARRIVAL FORFEIT DEPOSIT OF 3 NIGHTS ROOM AND TAX",
+      "ALL RESERVATIONS MUST BE CANCELLED 24 HOURS PRIOR TO HOST TIME UNLESS DEPOSIT REQUIRED IF THIS RESERVATION HAS BEEN MADE ELECTRONICALLY PLEASE CANCEL IT ELECTRONICALLY TO AVOID CONFUSION AND A NO SHOW BILL POLICY SUBJECT TO CHANGE .",
+      "PLEASE REFER TO RATE DETAILS AT THE TIME OF BOOKING FOR CANCELLATION POLICY. PLEASE NOTE THAT POLICY IS SUBJECT TO CHANGES WITHOUT NOTICE."
+    ]
+  end
+end

data/test/test_modules/unknown.rb

@@ -0,0 +1,9 @@
+module TeRex
+  module Train
+    UNKNOWN = [
+      "The cancellation policy will be determined when the rate is validated.",
+      "-CANCEL POLICY MAY VARY BY DAY OF WEEK AND SEASON. THE MOST ACCURATE CANCEL POLICY IS ADVISED DURING BOOKING PROCESS.  IN CASE OF A NO-SHOW THE CREDIT CARD WILL BE CHARGED ONE NIGHT STAY.  OUR SYSTEM ACKNOWLEDGES ALL PROPERLY CANCELED RESERVATIONS BY RETURNING A CANCELLATION NUMBER. DO NOT ASSUME YOUR RESERVATION IS CANCELED IF YOU HAVE NOT RECEIVED A CANCELLATION NUMBER IN YOUR PNR OR BOOKING FILE. IF YOU DO NOT RECEIVE A CANCELLATION NUMBER, PLEASE CALL THE CHOICE GDS DEPARTMENT AT 1-866-953-4570",
+      "-14JAN02 - END - CANCEL POLICIES VARY BY HOTEL.  SINCE A HOTEL CAN SET A CANCELLATION POLICY OF UP TO 30 DAYS IN ADVANCE, PLEASE REVIEW POLICY PRIOR TO BOOKING TO AVOID POSSIBLE CHARGE."
+    ]
+  end
+end

data/test/trained_bayes_test.rb

@@ -0,0 +1,140 @@
+require_relative "../lib/te_rex"
+class TrainedBayesTest < MicroTest::Test
+
+  #Dir["#{File.dirname(__FILE__)}/test_modules/**/*.rb"].each { |f| load(f) if !!(f =~ /^[^\.].+\.rb/)}
+
+  @@refund = TeRex::Train::REFUND
+  @@partrefund = TeRex::Train::PARTREFUND
+  @@norefund = TeRex::Train::NONREFUND
+  @@unknown = TeRex::Train::UNKNOWN
+
+  @@cls = TeRex::Classifier::Bayes.new("Refund", "Partrefund", "Nonrefund", "Unknown")
+  @@refund.each {|txt| @@cls.train("Refund", txt) }
+  @@partrefund.each {|txt| @@cls.train("Partrefund", txt) }
+  @@norefund.each {|txt| @@cls.train("Nonrefund", txt) }
+  @@unknown.each {|txt| @@cls.train("Unknown", txt) }
+
+  test "Training Data Set Test: Random exact match sould classify correctly (but we are lenient on partrefund/refund)" do
+
+    s_refund = @@refund.sample
+    s_partial = @@partrefund.sample
+    s_non = @@norefund.sample
+    s_unk = @@unknown.sample
+
+    s_refund1 = @@cls.classify(s_refund)
+    s_partial1 = @@cls.classify(s_partial)
+    s_non1 = @@cls.classify(s_non)
+    s_unk1= @@cls.classify(s_unk)
+
+    # We are lenient on Partrefund || Refund but we still want to see when it fails
+    assert s_refund1 == "Refund" || "Partrefund"
+    # We are lenient on Refund || Partrefund because of the non-distinctness of the two.
+    assert s_partial1 == "Partrefund" || "Refund"
+    assert s_non1 == "Nonrefund"
+    assert s_unk1 == "Unknown"
+
+    # We are lenient on Partrefund || Refund but we still want to see when it fails
+    assert s_refund1 != "Partrefund"
+    # We are lenient on Refund || Partrefund but we still want to see when it fails
+    assert s_partial1 != "Refund"
+    assert s_non1 != "Unknown"
+    assert s_unk1 != "Nonrefund"
+  end
+
+
+  test "Training Data Set Test: Non-canonical examples should classify correctly" do
+
+    refund_s1 = "You will get a full refund and free cancellation"
+    partrefund_s1 = "You will get a refund if you cancel or change your reservation before 0201 AM on 01/31/14"
+    norefund_s1 = "You will get a non-refund"
+    unk_s1 = "You will get a nonsense am I writing here."
+
+    refund_s11 = @@cls.classify(refund_s1)
+    partrefund_s11 = @@cls.classify(partrefund_s1)
+    norefund_s11 = @@cls.classify(norefund_s1)
+    unk_s11 = @@cls.classify(unk_s1)
+
+    assert refund_s11 == "Refund"
+    assert partrefund_s11 == "Partrefund"
+    assert norefund_s11 == "Nonrefund"
+    assert unk_s11 == "Unknown"
+  end
+
+  test "Training Data Set Test: Micro examples should return correct classification" do
+
+    s1 = "free cancellation"
+    s2 = "If you cancel or change your reservation before"
+    s3 = "non-refund"
+    s4 = "policy rate validated."
+
+    s11 = @@cls.classify(s1)
+    s22 = @@cls.classify(s2)
+    s33 = @@cls.classify(s3)
+    s44 = @@cls.classify(s4)
+
+    assert s11 == "Refund"
+    assert s22 == "Partrefund"
+    assert s33 == "Nonrefund"
+    assert s44 == "Unknown"
+
+    assert s11 != "Partrefund"
+    assert s22 != "Refund"
+    assert s33 != "Unknown"
+    assert s44 != "Nonrefund"
+  end
+
+  test "Training Data Set Test: Micro examples should NOT match fake classes" do
+
+    s1 = "free cancellation"
+    s2 = "partial refund"
+    s3 = "no refund"
+    s4 = "policy rate validated."
+
+    s11 = @@cls.classify(s1)
+    s22 = @@cls.classify(s2)
+    s33 = @@cls.classify(s3)
+    s44 = @@cls.classify(s4)
+
+    assert s11 != "Computers"
+    assert s22 != "Science"
+    assert s33 != "Entertainment"
+    assert s44 != "Sports"
+  end
+
+  test "Training Data Set Test: Ambiguous examples should return 'Unknown'" do
+
+    s1 = "gobbly goop droop blithely toadwakle Grimpleshtein uf Varendorrf vun muscilaty"
+    s2 = "The United States announced on Tuesday it will send 3,000 troops to help tackle the Ebola outbreak as part of a ramped-up plan, including a major deployment in Liberia."
+    s3 = "United Parcel Service Inc is almost doubling the number of seasonal employees it hires for this year's holiday shopping season as it aims to avoid a repeat of last year's network breakdown."
+    s4 = "Alberto Contador wrapped up his third Vuelta a España triumph when he comfortably held on to his overall lead in the 21st and final stage time trial in a rain-soaked Santiago de Compostela on Sunday."
+
+    s11 = @@cls.classify(s1)
+    s22 = @@cls.classify(s2)
+    s33 = @@cls.classify(s3)
+    s44 = @@cls.classify(s4)
+
+    assert s11 == "Unknown"
+    assert s22 == "Unknown"
+    assert s33 == "Unknown"
+    assert s44 == "Unknown"
+  end
+
+  test "Training Data Set Test: Category counts are equivalent with number of training data per class" do
+
+    assert @@cls.category_counts[:Refund] == @@refund.count 
+    assert @@cls.category_counts[:Partrefund] == @@partrefund.count 
+    assert @@cls.category_counts[:Nonrefund] == @@norefund.count 
+    assert @@cls.category_counts[:Unknown] == @@unknown.count 
+
+  end
+
+  test "Sparse Data Set Test: Training categories should NOT be undertrained... except 'Unknown'" do
+    info = @@cls.training_description
+    puts "\nUndertraining data for SPARSE DATA SET: #{info}"
+    res = @@cls.under_trained?
+    assert res[0].include? :Unknown
+  end
+
+end
+
+

metadata

@@ -0,0 +1,231 @@
+--- !ruby/object:Gem::Specification
+name: te_rex
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- Joshua Bowles
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fast-stemmer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+- !ruby/object:Gem::Dependency
+  name: micro_test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.4
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+- !ruby/object:Gem::Dependency
+  name: pry-debugger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.3
+- !ruby/object:Gem::Dependency
+  name: pry-rescue
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.1
+- !ruby/object:Gem::Dependency
+  name: pry-stack_explorer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.9.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.9.1
+description: Simple text processing for small data sets.
+email:
+- jbowayles@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/format/corpus/basic_file.rb
+- lib/format/corpus/brown_file.rb
+- lib/format/format.rb
+- lib/te_rex.rb
+- lib/te_rex/alpha_num.rb
+- lib/te_rex/bayes.rb
+- lib/te_rex/bayes_data.rb
+- lib/te_rex/corpus.rb
+- lib/te_rex/stop_word.rb
+- lib/te_rex/version.rb
+- test/alpha_num_test.rb
+- test/bayes_data_test.rb
+- test/sparse_bayes_test.rb
+- test/test_modules/nonrefund.rb
+- test/test_modules/partrefund.rb
+- test/test_modules/refund.rb
+- test/test_modules/unknown.rb
+- test/trained_bayes_test.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: Basic NLP stuff for small data sets. Naive Bayes classification and corpora
+  loading.
+test_files:
+- test/alpha_num_test.rb
+- test/bayes_data_test.rb
+- test/sparse_bayes_test.rb
+- test/test_modules/nonrefund.rb
+- test/test_modules/partrefund.rb
+- test/test_modules/refund.rb
+- test/test_modules/unknown.rb
+- test/trained_bayes_test.rb