unsupervised-language-detection 0.0.1

data/website/main.rb

@@ -0,0 +1,46 @@
+require 'sinatra'
+require 'uri'
+require 'mongo'
+require 'date'
+require 'time'
+require File.expand_path('../../src/language-detector', __FILE__)
+
+use Rack::MethodOverride
+
+# Tweets come from a MongoDB collection.
+uri = URI.parse(ENV['MONGOHQ_URL'])
+conn = Mongo::Connection.from_uri(ENV['MONGOHQ_URL'])
+db = conn.db(uri.path.gsub(/^\//, ''))
+coll = db["tweets"]
+
+DETECTOR = LanguageDetector.load_yaml("detector2.yaml")
+
+helpers do
+  def partial(page, locals = {})
+    haml page, :layout => false, :locals => locals
+  end  
+end
+
+layout 'layout'
+  
+get '/' do  
+  haml :index
+end
+
+post '/' do
+  @sentence = nil
+  if params[:sentence]
+    @sentence = params[:sentence]
+    @language = DETECTOR.classify(@sentence) == "majority" ? "English" : "Not English"
+  end
+  
+  haml :index
+end
+
+get '/tweet' do
+  @tweet = coll.find().limit(-1).skip(rand(coll.count())).first()['text']
+  @language = DETECTOR.classify(@tweet) == "majority" ? "English" : "Not English"
+  @language = "Not English" if @tweet.split.select{ |c| c =~ /[^\x00-\x80]/ }.size > 1 # Use this if you want to check for non-Roman characters. Not necessary, but sometimes there are tweets consisting solely of non-Roman characters, in which case the classifier fails (since it currently removes all non-ASCII characters).
+
+  haml :tweet, :layout => false
+end

data/website/public/jquery.inlineformlabels.js

@@ -0,0 +1,53 @@
+(function($) {
+
+	$.fn.inlineFormLabels = function() {
+		
+		var self = this;
+		
+		// Hide all the labels, because we're going to put them in the input field instead.
+		$("label", self).hide();
+		
+		// Grab all input fields (inputs and textareas) preceded by a label sibling... 
+		$("label + input, label + textarea", self).each(function(type) {
+			
+			// If the field is empty, display the label and add a class that indicates its holding the label.
+			if ($(this).val() == "") {
+				var labelText = $(this).prev("label, textarea").text().trim();
+				$(this).val(labelText).addClass("has-inline-label");
+			}
+			
+			// If we click in the field, remove the label.
+			$(this).focus(function() {
+				if ($(this).hasClass("has-inline-label")) {
+					$(this).removeClass("has-inline-label");
+					$(this).val("");
+				}
+			});
+			
+			// Not doing anything here yet...
+			$(this).keypress(function() {
+			});
+			
+			// If we click out of the field and we haven't entered anything, redisplay the label and add back the label-indicator class.
+			$(this).blur(function() {
+				if ($(this).val() == "") {
+					var labelText = $(this).prev("label").text().trim();
+					$(this).val(labelText).addClass("has-inline-label");
+				}
+			});
+			
+		});
+
+		// When submitting, remove the values from fields holding a label, so that we don't mistakenly think those are real inputs.
+		$(self).submit(function() {
+			$("input, textarea", self).each(function() {
+				if ($(this).hasClass("has-inline-label")) {
+					$(this).val("");
+				}
+			});
+		});
+		
+		return self;
+	};
+	
+})(jQuery);

data/website/public/main.css

@@ -0,0 +1,23 @@
+#container {
+	margin: 10px 20px;
+}
+
+table {
+	text-align: left;
+}
+
+	table th.language, table td.language {
+		width: 150px;
+	}
+
+.english {
+	background-color: #99FF99;
+}
+
+.other {
+	background-color: #FF9999;
+}
+
+form {
+	margin-bottom: 10px;
+}

data/website/views/index.haml

@@ -0,0 +1,36 @@
+%p An unsupervised language identification algorithm. Trained on tweets with lang = "en" according to the Twitter API (which, in practice, returns tweets in Spanish, Portuguese, Dutch, Russian, and a couple other languages as well). More information <a href="http://blog.echen.me/2011/05/01/unsupervised-language-detection-algorithms/">here</a>.
+
+%form{ :action => "/", :method => "post" }
+  %label{ :for => "sentence" } Sentence
+  %input{ :id => "sentence", :name => "sentence" }
+  %button{ :type => "submit" } Detect Language
+
+- if @sentence and !@sentence.empty?
+  %p
+    %strong= @sentence
+    is
+    %span{ :class => "#{@language == "English" ? "english" : "other"}"}= @language
+    
+%table
+  %tr#header
+    %th.language Language
+    %th.tweet Tweet
+
+:javascript
+  function addTweet() {
+    $.ajax({
+        method: 'GET',
+        url: '/tweet',
+        cache: false,
+        success: function(data) {
+            $("table tr#header:first").after(data).slideDown('slow');
+            setTimeout(addTweet, 500);                
+        }
+    });
+    $('table tr.tweet:gt(20)').remove();
+  }
+  
+  $(function() {
+    $("form").inlineFormLabels();
+    setTimeout(addTweet, 500);
+  });

data/website/views/layout.haml

@@ -0,0 +1,14 @@
+!!! 5
+%html
+  %head
+    %meta{ "http-equiv" => "Content-Type", :content => "text/html", :charset => "UTF-8" }  
+    
+    %title Babel Fett    
+    %script{ :type => "text/javascript", :src => "http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js" }    
+    %script{ :type => "text/javascript", :src => "/jquery.inlineformlabels.js" }    
+    %link{ :href => "/main.css", :rel => "stylesheet", :type => "text/css" }    
+    
+  %body  
+    #container
+      %h1 Unsupervised Language Detection on Twitter    
+      = yield

data/website/views/tweet.haml

@@ -0,0 +1,3 @@
+%tr{:class => "tweet #{@language == "English" ? "english" : "other"}"}
+  %td.language= @language
+  %td.text= @tweet

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification 
+name: unsupervised-language-detection
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Edwin Chen
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-14 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Perform unsupervised language detection, specifically for the purpose of finding English-language tweets.
+email: 
+- hello@echen.me
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Gemfile
+- README.md
+- Rakefile
+- datasets/gutenberg-test-du.txt
+- datasets/gutenberg-test-en.txt
+- datasets/gutenberg-test-sp.txt
+- datasets/gutenberg-training-du.txt
+- datasets/gutenberg-training-en.txt
+- datasets/gutenberg-training-sp.txt
+- datasets/gutenberg-training.txt
+- datasets/gutenberg-training_en_du.txt
+- datasets/smiley_tweets_tiny.txt
+- datasets/tweets_5000.txt
+- language-detector-demo.rb
+- lib/unsupervised-language-detection.rb
+- lib/unsupervised-language-detection/english-tweet-detector.yaml
+- lib/unsupervised-language-detection/language-detector.rb
+- lib/unsupervised-language-detection/naive-bayes-classifier.rb
+- lib/unsupervised-language-detection/train-english-tweet-detector.rb
+- lib/unsupervised-language-detection/version.rb
+- test/test_language_detector.rb
+- test/test_naive_bayes_classifier.rb
+- test/test_naive_bayes_em.rb
+- test/test_suite.rb
+- unsupervised-language-detection.gemspec
+- website/Gemfile
+- website/README.md
+- website/config.ru
+- website/detector.yaml
+- website/detector2.yaml
+- website/main.rb
+- website/public/jquery.inlineformlabels.js
+- website/public/main.css
+- website/views/index.haml
+- website/views/layout.haml
+- website/views/tweet.haml
+has_rdoc: true
+homepage: http://blog.echen.me/2011/05/01/unsupervised-language-detection-algorithms/
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: unsupervised-language-detection
+rubygems_version: 1.4.1
+signing_key: 
+specification_version: 3
+summary: Perform unsupervised language detection.
+test_files: 
+- test/test_language_detector.rb
+- test/test_naive_bayes_classifier.rb
+- test/test_naive_bayes_em.rb
+- test/test_suite.rb