jakal 0.1.6 → 0.1.7

data/lib/jkl/calais_client.rb

@@ -11,6 +11,10 @@ module Jkl
         )
       end
 
+      def entities(key,text)
+        calais_response(key, text).entities.map{|e| {e.type => [e.attributes["name"]]}}
+      end
+
       def tags(key, text)
         nested_list = {}
         entities(key,text).each do |a| 
@@ -19,10 +23,6 @@ module Jkl
         nested_list
       end
 
-      def entities(key,text)
-        calais_response(key, text).entities.map{|e| {e.type => [e.attributes["name"]]}}
-      end
-
     end
   end
 end

data/lib/jkl/text_client.rb

@@ -5,6 +5,7 @@ module Jkl
       def sanitize(text)
         remove_short_lines(strip_all_tags(remove_script_tags(text)))
       end
+      alias :clean :sanitize
 
       def strip_all_tags(text)
         text.gsub(/<\/?[^>]*>/, "")

data/lib/jkl.rb

@@ -16,10 +16,19 @@ module Jkl
       end
     end
     
+    def topix_links(keyphrase)
+      url = YAML::load_file('config/config.yml')['topix']
+      links("#{url}#{keyphrase}")
+    end
+    
     def tags(key, link)
       text = Jkl::Text::sanitize(Jkl::get_from(link))
       Jkl::Extraction::tags(key, text)
     end
 
+    def trends
+      url = YAML::load_file('config/config.yml')['twitter']
+      JSON.parse(Jkl::get_from(url))["trends"].map{|t| t["name"]}
+    end
   end
 end

data/test/unit/jkl_test.rb

@@ -0,0 +1,82 @@
+require "test/unit"
+require "shoulda"
+require "webmock/test_unit"
+require "yaml"
+require "lib/jkl"
+
+class JklTest < Test::Unit::TestCase
+  include WebMock
+
+  context "Using Jkl" do    
+    setup do
+      stub_twitter
+      stub_topix
+      stub_news_article
+    end
+    
+    should "GET trends" do
+      trends = Jkl::trends
+      assert trends.length == 10
+      assert trends[0] == "London"
+    end
+    
+    should "GET news article URLS for a trend" do
+      articles = Jkl::topix_links(Jkl::trends[0])
+      assert articles.length == 2
+      assert articles[0] = "http://www.localnews8.com/Global/story.asp?S=10876507"
+    end
+    
+    should "extract text from a news article" do
+      articles = Jkl::topix_links(Jkl::trends[0])
+      text = Jkl::Text::sanitize(Jkl::get_from(articles[0]))
+      assert_not_nil text
+    end
+    
+    should "extract tags from some text" do
+      key = YAML::load_file('config/keys.yml')['calais']
+      text = <<-EOF
+        Barack Obama said today that he expects there 
+        to be conflict within his new security team after 
+        confirming Hillary Clinton as his choice for US Secretary of State."
+      EOF
+      tags = Jkl::Extraction::tags(key, text)
+      assert tags["Person"][0] == "Barack Obama"
+      puts Jkl::Extraction::entities(key,text)
+    end
+  end
+  
+  private 
+  def stub_twitter
+    url = YAML::load_file('config/config.yml')['twitter']
+    response = <<-EOF
+    {"trends":[
+      {"name":"London","url":"http://search.twitter.com/search?q=London"},
+      {"name":"Geneva","url":"http://search.twitter.com/search?q=Geneva"},
+      {"name":"Kabul","url":"http://search.twitter.com/search?q=Kabul"},
+      {"name":"Chicago","url":"http://search.twitter.com/search?q=Chicago"},
+      {"name":"Cannes","url":"http://search.twitter.com/search?q=Cannes"},
+      {"name":"Verona","url":"http://search.twitter.com/search?q=Verona"},
+      {"name":"Milan","url":"http://search.twitter.com/search?q=Milan"},
+      {"name":"New York","url":"http://search.twitter.com/search?q=New%20York"},
+      {"name":"Paris","url":"http://search.twitter.com/search?q=Paris"},
+      {"name":"Melbourne","url":"http://search.twitter.com/search?q=Melbourne"}
+      ],"as_of":"Sat, 1 Jan 1970 00:00:00 +0000"}
+    EOF
+    stub_request(:get, url).to_return(:body => response)
+  end
+  def stub_topix
+    url = YAML::load_file('config/config.yml')['topix']
+    response = raw = File.open('test/fixtures/topix_rss.xml','r') do |file| 
+      file.readlines.to_s
+    end
+    stub_request(:get, "#{url}London").to_return(:body => response)
+  end  
+  def stub_news_article
+    response = raw = File.open('test/fixtures/bbc_story.html','r') do |file| 
+      file.readlines.to_s
+    end
+    stub_request(:get, "http://www.localnews8.com/Global/story.asp?S=10876507").to_return(
+      :body => response
+    )
+  end
+end

data/test/unit/text_cleaning_test.rb

@@ -0,0 +1,72 @@
+require "test/unit"
+require "shoulda"
+require "webmock/test_unit"
+require "yaml"
+require "lib/jkl"
+
+class TextCleaningTest < Test::Unit::TestCase
+  context "Cleaning Text" do
+    
+    should "Remove short lines" do
+      input = <<-HTML
+the cat sat on the mat
+a short line
+HTML
+      result = Jkl::Text::remove_short_lines input
+      assert result == "the cat sat on the mat"
+    end
+    
+    should "Remove script tags" do
+      input = <<-HTML
+the cat sat on the mat
+<script type="text/javascript" charset="utf-8">
+function nofunction(){var bob;}
+</script>
+a short line
+HTML
+      result = Jkl::Text::remove_short_lines input
+      assert result == "the cat sat on the mat"
+    end
+    
+    should "Remove html comments" do
+      input = <<-HTML
+the cat sat on the mat
+<!-- a comment-->
+a short line
+HTML
+      result = Jkl::Text::remove_short_lines input
+      assert result == "the cat sat on the mat"      
+    end
+    
+    should "Remove blank lines" do
+      input = <<-HTML
+the cat sat on the mat
+
+a short line
+HTML
+      result = Jkl::Text::remove_short_lines input
+      assert result == "the cat sat on the mat"
+    end
+    
+    should "Strip all tags" do
+      input = <<-HTML
+<p>the cat sat on the mat</p>
+HTML
+      result = Jkl::Text::strip_all_tags input
+      assert result == "the cat sat on the mat\n" #TODO fix carriage return
+    end
+  
+    should "Clean text" do
+      input = <<-HTML
+the cat sat on the mat
+<script type="text/javascript" charset="utf-8">
+ function nofunction(){var bob;}
+</script>
+<p> some para stuff here </p>
+some end stuff here
+HTML
+      result = Jkl::Text::clean input
+      assert result == "the cat sat on the mat"
+    end
+  end
+end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 6
-  version: 0.1.6
+  - 7
+  version: 0.1.7
 platform: ruby
 authors: 
 - sshingler
@@ -73,7 +73,7 @@ dependencies:
         version: 0.0.9
   type: :runtime
   version_requirements: *id004
-description: Jakal is a Ruby library which contains some utilities for connecting to internet based APIs.
+description: Jakal is a Ruby library which contains some utilities for connecting to internet based APIs and cleaning text.
 email: "'shingler@gmail.com'"
 executables: []
 
@@ -88,20 +88,13 @@ files:
 - lib/jkl/rest_client.rb
 - lib/jkl/rss_client.rb
 - lib/jkl/text_client.rb
-- features/calais.feature
-- features/http.feature
-- features/sanitize-text.feature
-- features/mocks/bbc_story.html
-- features/mocks/calais.json
-- features/mocks/topix_rss.xml
-- features/step_definitions/calais_steps.rb
-- features/step_definitions/http_steps.rb
-- features/step_definitions/sanitize-text_steps.rb
-- features/step_definitions/twitter_steps.rb
-- features/support/env.rb
+- test/fixtures/bbc_story.html
+- test/fixtures/topix_rss.xml
+- test/unit/jkl_test.rb
+- test/unit/text_cleaning_test.rb
 - README.md
 - License.txt
-has_rdoc: false
+has_rdoc: true
 homepage: http://github.com/sshingler/jkl
 licenses: []
 
@@ -131,6 +124,6 @@ rubyforge_project:
 rubygems_version: 1.3.6
 signing_key: 
 specification_version: 2
-summary: Jakal is a Ruby library which contains some utilities for connecting to internet based APIs.
+summary: Jakal is a Ruby library which contains some utilities for connecting to internet based APIs and cleaning text.
 test_files: []
 

data/features/calais.feature

@@ -1,10 +0,0 @@
-Feature: Calais-Specific features
-  In order to use the Calais Meta-Tagging Service
-  As a developer
-  I want to make some requests and inspect some responses
-
-  @live
-  Scenario: Get nested tags from calais
-    Given I have some text
-    When I request the nested entities from calais
-    Then I should receive the entities grouped into categories

data/features/http.feature

@@ -1,20 +0,0 @@
-Feature: http features
-  In order to use the utility client calls
-  As a developer
-  I want to make some requests and inspect some responses
-
-  @live
-  Scenario: Make a restful post to yahoo
-    When I post some data to yahoo
-	  Then I should get a response
-
-  @live
-  Scenario: Make a restful get
-    When I make a restful get request
-	  Then I should get a response
-	  And I should see some text
-
-  @live
-  Scenario: Get some trends
-    When I request some twitter trends
-	  Then I should get a response