jakal 0.1.96 → 0.2.0

data/.gitignore

@@ -0,0 +1,7 @@
+*.gem
+keys.yml
+coverage
+.bundle
+webrat.log
+._*
+.DS*

data/Gemfile

@@ -0,0 +1,13 @@
+source :gemcutter
+
+gemspec
+
+group :development do
+  gem "ruby-debug19", "0.11.6", :require => nil
+end  
+
+group :test do
+  gem "test-unit", "2.3.0", :require => "test/unit"  
+  gem "shoulda", "2.11.3"
+  gem "webmock", "1.6.2"
+end

data/Gemfile.lock

@@ -0,0 +1,57 @@
+PATH
+  remote: .
+  specs:
+    jakal (0.2.0)
+      calais (>= 0.0.11)
+      mechanize (>= 1.0.0)
+      nokogiri (~> 1.4.4)
+      rake (>= 0.8.7)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.2.6)
+    archive-tar-minitar (0.5.2)
+    calais (0.0.11)
+      curb (>= 0.1.4)
+      json (>= 1.1.3)
+      nokogiri (>= 1.3.3)
+    columnize (0.3.2)
+    crack (0.1.8)
+    curb (0.7.15)
+    json (1.5.1)
+    linecache19 (0.5.12)
+      ruby_core_source (>= 0.1.4)
+    mechanize (1.0.0)
+      nokogiri (>= 1.2.1)
+    nokogiri (1.4.4)
+    rake (0.8.7)
+    ruby-debug-base19 (0.11.25)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby_core_source (>= 0.1.4)
+    ruby-debug19 (0.11.6)
+      columnize (>= 0.3.1)
+      linecache19 (>= 0.5.11)
+      ruby-debug-base19 (>= 0.11.19)
+    ruby_core_source (0.1.5)
+      archive-tar-minitar (>= 0.5.2)
+    shoulda (2.11.3)
+    test-unit (2.3.0)
+    webmock (1.6.2)
+      addressable (>= 2.2.2)
+      crack (>= 0.1.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  calais (>= 0.0.11)
+  jakal!
+  mechanize (>= 1.0.0)
+  nokogiri (~> 1.4.4)
+  rake (>= 0.8.7)
+  ruby-debug19 (= 0.11.6)
+  shoulda (= 2.11.3)
+  test-unit (= 2.3.0)
+  webmock (= 1.6.2)

data/Rakefile

@@ -0,0 +1,7 @@
+require 'rubygems'
+require 'rake/testtask'
+
+desc "Run all the tests"
+Rake::TestTask.new do |t|
+    t.test_files = FileList['test/unit/*.rb']
+end

data/jkl.gemspec

@@ -0,0 +1,19 @@
+Gem::Specification.new do |s|
+  s.name = %q{jakal}
+  s.version = "0.2.0"
+  s.platform = Gem::Platform::RUBY
+  s.authors = ['sshingler']
+  s.homepage = %q{http://github.com/sshingler/jkl}
+  s.description = %q{Jakal is a Ruby library for tagging keywords from web pages.}
+  s.summary = s.description
+  s.email = %q{'shingler@gmail.com'}
+  
+  s.require_paths = ["lib"]
+  s.files = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+
+  s.add_dependency(%q<rake>, [">= 0.8.7"])
+  s.add_dependency(%q<mechanize>, [">= 1.0.0"])
+  s.add_dependency(%q<nokogiri>, ["~> 1.4.4"])
+  s.add_dependency(%q<calais>, [">= 0.0.11"])
+end

data/lib/jkl/rss_client.rb

@@ -1,27 +1,15 @@
-require 'hpricot'
 
 module Jkl
   module Rss
     class << self
-
-      def items(rss_doc)
-        (rss_doc/:item)
+      def items(rss)
+        rss_doc = Nokogiri::Slop(rss)
+        rss_doc.rss.channel.item
       end
       
       def links(items)
-        items.map{|item| attribute_from(item,:link)}
-      end
-
-      def descriptions(items)
-        items.map do |item| 
-          attribute_from(item, :description).gsub("<![CDATA[","").gsub("]]>","")
-        end
+        items.map{|item| item.link.inner_html}
       end
-
-      def attribute_from(item, name)
-        (item/name).inner_html
-      end
-      
     end
   end
 end

data/lib/jkl/text_client.rb

@@ -2,10 +2,9 @@ module Jkl
   module Text
     class << self
 
-      def sanitize(text, words_on_line = 5)
-        remove_short_lines(strip_all_tags(remove_script_tags(text)), words_on_line)
+      def plain_text(document, words_on_line = 5)
+        remove_short_lines(strip_all_tags(remove_script_tags(document)), words_on_line)
       end
-      alias :clean :sanitize
 
       def strip_all_tags(text)
         text.gsub(/<\/?[^>]*>/, "")

data/lib/jkl.rb

@@ -1,32 +1,22 @@
-$:.unshift(File.dirname(__FILE__)) unless
-  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+require_relative "jkl/rss_client"
+require_relative "jkl/calais_client"
+require_relative "jkl/text_client"
 
-require "jkl/rss_client"
-require "jkl/rest_client"
-require "jkl/calais_client"
-require "jkl/text_client"
+require "mechanize"
 
 module Jkl
   class << self
     
+    def get(url)
+      agent = Mechanize.new
+      agent.get(url).body
+    end
+    
     def links(feed)
-      links = Jkl::Rss::links(Jkl::Rss::items(Jkl::get_xml_from(feed)))
+      links = Jkl::Rss::links(Jkl::Rss::items(Jkl::get(feed)))
       links.each do |link|
         yield link if block_given?
       end
     end
-    
-    def topix_links(keyphrase, url = "http://www.topix.net/rss/search/article?q=")
-      links("#{url}#{keyphrase}")
-    end
-    
-    def tags(key, link)
-      text = Jkl::Text::sanitize(Jkl::get_from(link))
-      Jkl::Extraction::tags(key, text)
-    end
-
-    def trends(url = "http://search.twitter.com/trends.json")
-      JSON.parse(Jkl::get_from(url))["trends"].map{|t| t["name"]}
-    end
   end
 end

data/test/unit/jkl_test.rb

@@ -2,78 +2,56 @@ require "test/unit"
 require "shoulda"
 require "webmock/test_unit"
 require "yaml"
+
 require_relative "../../lib/jkl"
 
 class JklTest < Test::Unit::TestCase
   include WebMock::API
 
-  context "Using Jkl" do    
+  context "for documents, plain text and tags" do
     setup do
-      stub_twitter
-      stub_topix
-      stub_news_article
-    end
-    
-    should "GET trends" do
-      trends = Jkl::trends
-      assert trends.length == 10
-      assert trends[0] == "London"
+      @url = "http://www.bbc.co.uk"
+      response = File.read('test/fixtures/bbc_story.html')
+      stub_request(:get, @url).to_return(
+          :status => 200, 
+          :body => response, 
+          :headers => {'Content-Type' => 'text/html'})
     end
     
-    should "GET news article URLS for a trend" do
-      articles = Jkl::topix_links(Jkl::trends[0])
-      assert articles.length == 2
-      assert articles[0] = "http://www.localnews8.com/Global/story.asp?S=10876507"
+    should "Get a document from a URL" do
+      doc = Jkl::get(@url)
+      assert_not_nil doc
     end
-    
-    should "extract text from a news article" do
-      articles = Jkl::topix_links(Jkl::trends[0])
-      text = Jkl::Text::sanitize(Jkl::get_from(articles[0]))
-      assert_not_nil text
+  
+    should "Get the plain text version of a document" do
+      document = Jkl::get(@url)
+      text = Jkl::Text::plain_text(document,2)
+      assert_equal 8884, text.length
     end
-    
-    should "extract tags from some text" do
-      keys = "config/keys.yml"
-      raise "READ:::::::: You need to create #{keys} and put your calais credentials in it." unless File.exist?(keys)
-      key = YAML::load_file(keys)['calais']
-      text = <<-EOF
-        Barack Obama said today that he expects there 
-        to be conflict within his new security team after 
-        confirming Hillary Clinton as his choice for US Secretary of State."
-      EOF
-      tags = Jkl::Extraction::tags(key, text)
-      assert tags["Person"][0] == "Barack Obama"
+  
+    should "Get the keywords from a document" do
+      document = Jkl::get(@url)
+      text = Jkl::Text::plain_text(document,2)
+      tags = Jkl::Extraction::tags(calais_key, text)
+      assert ! tags.empty?
     end
   end
   
-  private 
-  def stub_twitter
-    url = YAML::load_file('config/config.yml')['twitter']
-    response = <<-EOF
-    {"trends":[
-      {"name":"London","url":"http://search.twitter.com/search?q=London"},
-      {"name":"Geneva","url":"http://search.twitter.com/search?q=Geneva"},
-      {"name":"Kabul","url":"http://search.twitter.com/search?q=Kabul"},
-      {"name":"Chicago","url":"http://search.twitter.com/search?q=Chicago"},
-      {"name":"Cannes","url":"http://search.twitter.com/search?q=Cannes"},
-      {"name":"Verona","url":"http://search.twitter.com/search?q=Verona"},
-      {"name":"Milan","url":"http://search.twitter.com/search?q=Milan"},
-      {"name":"New York","url":"http://search.twitter.com/search?q=New%20York"},
-      {"name":"Paris","url":"http://search.twitter.com/search?q=Paris"},
-      {"name":"Melbourne","url":"http://search.twitter.com/search?q=Melbourne"}
-      ],"as_of":"Sat, 1 Jan 1970 00:00:00 +0000"}
-    EOF
-    stub_request(:get, url).to_return(:body => response)
-  end
-  def stub_topix
-    url = YAML::load_file('config/config.yml')['topix']
-    response = File.read('test/fixtures/topix_rss.xml')
-    stub_request(:get, "#{url}London").to_return(:body => response)
-  end  
-  def stub_news_article
-    response = File.read('test/fixtures/bbc_story.html')
-    stub_request(:get, "http://www.localnews8.com/Global/story.asp?S=10876507").to_return(
-      :body => response
-    )
+  context "for RSS" do
+    should "Get links from a feed" do
+      feed = "http://feeds.bbci.co.uk/news/rss.xml"
+      response = File.read('test/fixtures/topix_rss.xml')
+      stub_request(:get, "http://feeds.bbci.co.uk/news/rss.xml").
+          to_return(:status => 200, :body => response, :headers => {})
+      first_link = "http://www.localnews8.com/Global/story.asp?S=10876507"
+      assert_equal first_link, Jkl::links(feed).first
+    end
   end
-end
+  
+  private
+    def calais_key
+      keys = "config/keys.yml"
+      raise "READ:::::::: You need to create #{keys} and put your calais credentials in it." unless File.exist?(keys)
+      YAML::load_file(keys)['calais']
+    end
+end

data/test/unit/text_cleaning_test.rb

@@ -75,7 +75,7 @@ the cat sat on the mat
 <p> some para stuff here </p>
 some end stuff here
 HTML
-      result = Jkl::Text::clean input
+      result = Jkl::Text::plain_text(input)
       assert result == "the cat sat on the mat"
     end
   end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
-  - 96
-  version: 0.1.96
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - sshingler
@@ -14,11 +14,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-13 00:00:00 +00:00
+date: 2011-05-28 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: hpricot
+  name: rake
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
@@ -28,12 +28,12 @@ dependencies:
         segments: 
         - 0
         - 8
-        - 2
-        version: 0.8.2
+        - 7
+        version: 0.8.7
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: json
+  name: mechanize
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
@@ -42,24 +42,24 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 1
-        - 2
-        - 4
-        version: 1.2.4
+        - 0
+        - 0
+        version: 1.0.0
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: rest-client
+  name: nokogiri
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
         segments: 
         - 1
         - 4
-        - 2
-        version: 1.4.2
+        - 4
+        version: 1.4.4
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
@@ -73,40 +73,41 @@ dependencies:
         segments: 
         - 0
         - 0
-        - 9
-        version: 0.0.9
+        - 11
+        version: 0.0.11
   type: :runtime
   version_requirements: *id004
-description: Jakal is a Ruby library which contains some utilities for tagging content, cleaning text from web pages and working with RSS feeds.
+description: Jakal is a Ruby library for tagging keywords from web pages.
 email: "'shingler@gmail.com'"
 executables: []
 
 extensions: []
 
-extra_rdoc_files: 
-- README.md
-- License.txt
+extra_rdoc_files: []
+
 files: 
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- License.txt
+- README.md
+- Rakefile
+- jkl.gemspec
 - lib/jkl.rb
 - lib/jkl/calais_client.rb
-- lib/jkl/rest_client.rb
 - lib/jkl/rss_client.rb
 - lib/jkl/text_client.rb
 - test/fixtures/bbc_story.html
 - test/fixtures/topix_rss.xml
 - test/unit/jkl_test.rb
 - test/unit/text_cleaning_test.rb
-- config/config.yml
-- README.md
-- License.txt
 has_rdoc: true
 homepage: http://github.com/sshingler/jkl
 licenses: []
 
 post_install_message: 
-rdoc_options: 
-- --inline-source
-- --charset=UTF-8
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -130,7 +131,10 @@ requirements: []
 rubyforge_project: 
 rubygems_version: 1.3.7
 signing_key: 
-specification_version: 2
-summary: Jakal is a Ruby library which contains some utilities for tagging content, cleaning text from web pages and working with RSS feeds.
-test_files: []
-
+specification_version: 3
+summary: Jakal is a Ruby library for tagging keywords from web pages.
+test_files: 
+- test/fixtures/bbc_story.html
+- test/fixtures/topix_rss.xml
+- test/unit/jkl_test.rb
+- test/unit/text_cleaning_test.rb

data/config/config.yml

@@ -1,4 +0,0 @@
-topix:
-    http://www.topix.net/rss/search/article?q=
-twitter:
-    http://search.twitter.com/trends.json

data/lib/jkl/rest_client.rb

@@ -1,49 +0,0 @@
-require 'net/http' 
-require 'hpricot'
-
-module Jkl
-  class << self
-    
-    def post_to(url, post_args = {})
-      begin
-        uri = URI.parse(url)
-        resp, data = Net::HTTP.post_form(uri, post_args)
-        data
-      rescue  URI::InvalidURIError => e
-        puts("WARN: JKL Invalid URI: #{e}")
-      rescue SocketError => e
-        puts("WARN: JKL Could not connect: #{e}")
-      rescue Errno::ECONNREFUSED  => e
-        puts("WARN: JKL Connection refused: #{e}")
-      end
-    end
-  
-    def get_from(uri)
-      begin
-        response = Net::HTTP.get_response(URI.parse(uri))
-        response.body
-      rescue  URI::InvalidURIError => e
-        puts("WARN: JKL Invalid URI: #{e}")
-      rescue SocketError => e
-        puts("WARN: JKL Could not connect: #{e}")
-      rescue Errno::ECONNREFUSED  => e
-        puts("WARN: JKL Connection refused: #{e}")
-      end
-    end
-    
-    def get_from_over_https(host, path)
-      http = Net::HTTP.new(host, "443")
-      http.use_ssl = true
-      http.get2(path) # returns [status, data]
-    end
-  
-    def get_xml_from(uri)
-      Hpricot.XML(get_from(uri))
-    end
-    
-    def document_from(text)
-      Hpricot(text)
-    end
-  
-  end
-end