unsupervised-language-detection 0.0.2 → 0.0.3

data/lib/unsupervised-language-detection/language-detector.rb

@@ -4,11 +4,13 @@ require File.expand_path('../naive-bayes-classifier', __FILE__)
 class String  
   # Returns a set of `n`-grams computed from this string.
   def to_ngrams(n)
-    self.normalize.scan(/.{#{n}}/)
+    self.normalize_tweet.scan(/.{#{n}}/)
   end
   
+  private
+  
   # TODO: Try not normalizing out all non-ASCII characters! Should significantly reduce false positive rate.
-  def normalize
+  def normalize_tweet
     self.remove_tweeters.remove_links.remove_hashtags.downcase.gsub(/\s/, " ").gsub(/[^a-z0-9\s]/, "")    
   end  
   

data/lib/unsupervised-language-detection/naive-bayes-classifier.rb

@@ -1,13 +1,3 @@
-class Array
-  def sum
-    self.reduce(0) { |total, element| total + element }
-  end
-  
-  def product
-    self.reduce(1) { |total, element| total * element }
-  end
-end
-
 class NaiveBayesClassifier
   attr_reader :num_categories, :prior_token_count, :prior_category_counts
   attr_accessor :category_names
@@ -72,10 +62,10 @@ class NaiveBayesClassifier
   # Returns p(category | token), for each category, in an array.
   def get_posterior_category_probabilities(tokens)
     unnormalized_posterior_probs = (0..@num_categories-1).map do |category|
-      p = tokens.map { |token| get_token_probability(token, category) }.product # p(tokens | category)
+      p = tokens.map { |token| get_token_probability(token, category) }.reduce(:*) # p(tokens | category)
       p * get_prior_category_probability(category) # p(tokens | category) * p(category)
     end
-    normalization = unnormalized_posterior_probs.sum
+    normalization = unnormalized_posterior_probs.reduce(:+)
     normalization = 1 if normalization == 0
     return unnormalized_posterior_probs.map{ |p| p / normalization }
   end    
@@ -92,7 +82,7 @@ class NaiveBayesClassifier
   
   # p(category)
   def get_prior_category_probability(category_index)
-    denom = @category_counts.sum + @prior_category_counts.sum    
+    denom = @category_counts.reduce(:+) + @prior_category_counts.reduce(:+)
     if denom == 0
       return 0
     else

data/lib/unsupervised-language-detection/version.rb

@@ -1,3 +1,3 @@
 module UnsupervisedLanguageDetection
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/website/views/layout.haml

@@ -11,4 +11,9 @@
   %body  
     #container
       %h1 Unsupervised Language Detection on Twitter    
-      = yield
+      = yield
+      
+  %footer
+    %p 
+      %strong How does this work?
+      Learn more at <a href = "http://blog.echen.me/2011/05/05/twss-building-a-thats-what-she-said-classifier/">here</a>. By <a href="http://echen.me">Edwin Chen</a>.

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: unsupervised-language-detection
 version: !ruby/object:Gem::Version 
-  hash: 27
-  prerelease: 
+  prerelease: false
   segments: 
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 3
+  version: 0.0.3
 platform: ruby
 authors: 
 - Edwin Chen
@@ -15,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-14 00:00:00 -07:00
+date: 2011-07-22 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -79,7 +78,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
@@ -88,14 +86,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: unsupervised-language-detection
-rubygems_version: 1.4.1
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Perform unsupervised language detection.