jakal 0.2.0 → 0.2.1

data/Gemfile.lock

@@ -2,30 +2,25 @@ PATH
   remote: .
   specs:
     jakal (0.2.0)
-      calais (>= 0.0.11)
       mechanize (>= 1.0.0)
       nokogiri (~> 1.4.4)
       rake (>= 0.8.7)
+      term-extract (~> 0.5.1)
 
 GEM
   remote: http://rubygems.org/
   specs:
     addressable (2.2.6)
     archive-tar-minitar (0.5.2)
-    calais (0.0.11)
-      curb (>= 0.1.4)
-      json (>= 1.1.3)
-      nokogiri (>= 1.3.3)
     columnize (0.3.2)
     crack (0.1.8)
-    curb (0.7.15)
-    json (1.5.1)
     linecache19 (0.5.12)
       ruby_core_source (>= 0.1.4)
     mechanize (1.0.0)
       nokogiri (>= 1.2.1)
     nokogiri (1.4.4)
     rake (0.8.7)
+    rbtagger (0.4.7)
     ruby-debug-base19 (0.11.25)
       columnize (>= 0.3.1)
       linecache19 (>= 0.5.11)
@@ -37,6 +32,8 @@ GEM
     ruby_core_source (0.1.5)
       archive-tar-minitar (>= 0.5.2)
     shoulda (2.11.3)
+    term-extract (0.5.1)
+      rbtagger
     test-unit (2.3.0)
     webmock (1.6.2)
       addressable (>= 2.2.2)
@@ -46,12 +43,12 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  calais (>= 0.0.11)
   jakal!
   mechanize (>= 1.0.0)
   nokogiri (~> 1.4.4)
   rake (>= 0.8.7)
   ruby-debug19 (= 0.11.6)
   shoulda (= 2.11.3)
+  term-extract (~> 0.5.1)
   test-unit (= 2.3.0)
   webmock (= 1.6.2)

data/README.md

@@ -1,59 +1,7 @@
 # jkl
 
-jkl (Jakal) does these things:
-
-* Connects to URLs.
-* Gets stuff out of RSS feeds.
-* Gets the main content from web pages
-* Gets a set of metadata from a web page (using the calais gem)
-
-# Sample usage
-
-For example - if you had a RSS feed:
-
-  require "jkl"
-  
-		feed = "http://www.topix.net/rss/search/article?x=0&y=0&q=London"
-
-You could collect some metadata from the links in that feed, thus:
-  
-		tags = []
-		Jkl::links(feed).each do |link|
-			tags << Jkl::tags("my_calais_key",link)
-		end
-  
-A metadata sample might look something like this:
-
-		{
-				"Person"=>["Barack Obama", "Hillary Clinton"], 
-				"Position"=>["Secretary of State"]
-		}
+Jakal is a Ruby library for tagging keywords from web pages.
   
 It is hosted at [gemcutter](http://gemcutter.org/gems/jakal)
 
-		gem install jakal
-
-# LICENSE:
-
-(The MIT License)
-
-Copyright (c) 2009 sshingler
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-'Software'), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
-SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+        gem install jakal

data/bin/convert_to_should_syntax

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'convert_to_should_syntax' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", __FILE__)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('shoulda', 'convert_to_should_syntax')

data/bin/edit_json.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'edit_json.rb' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", __FILE__)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('json', 'edit_json.rb')

data/bin/minitar

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'minitar' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", __FILE__)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('archive-tar-minitar', 'minitar')

data/bin/nokogiri

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'nokogiri' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", __FILE__)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('nokogiri', 'nokogiri')

data/bin/prettify_json.rb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'prettify_json.rb' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", __FILE__)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('json', 'prettify_json.rb')

data/bin/rake

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", __FILE__)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('rake', 'rake')

data/bin/rdebug

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'rdebug' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", __FILE__)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('ruby-debug19', 'rdebug')

data/bin/term-extract

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'term-extract' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", __FILE__)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('term-extract', 'term-extract')

data/bin/testrb

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'testrb' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile", __FILE__)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('test-unit', 'testrb')

data/jkl.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = %q{jakal}
-  s.version = "0.2.0"
+  s.version = "0.2.1"
   s.platform = Gem::Platform::RUBY
   s.authors = ['sshingler']
   s.homepage = %q{http://github.com/sshingler/jkl}
@@ -15,5 +15,5 @@ Gem::Specification.new do |s|
   s.add_dependency(%q<rake>, [">= 0.8.7"])
   s.add_dependency(%q<mechanize>, [">= 1.0.0"])
   s.add_dependency(%q<nokogiri>, ["~> 1.4.4"])
-  s.add_dependency(%q<calais>, [">= 0.0.11"])
+  s.add_dependency(%q<term-extract>, ["~> 0.5.1"])
 end

data/lib/jkl.rb

@@ -1,8 +1,8 @@
 require_relative "jkl/rss_client"
-require_relative "jkl/calais_client"
 require_relative "jkl/text_client"
 
 require "mechanize"
+require "term-extract"
 
 module Jkl
   class << self
@@ -18,5 +18,9 @@ module Jkl
         yield link if block_given?
       end
     end
+    
+    def tags(text)
+      TermExtract.extract(text.force_encoding("UTF-8")).keys
+    end
   end
 end

data/lib/jkl/text_client.rb

@@ -3,8 +3,12 @@ module Jkl
     class << self
 
       def plain_text(document, words_on_line = 5)
-        remove_short_lines(strip_all_tags(remove_script_tags(document)), words_on_line)
+        CGI::unescapeHTML(
+            remove_short_lines(
+                strip_all_tags(
+                    remove_script_tags(document)), words_on_line))
       end
+      alias :sanitize :plain_text
 
       def strip_all_tags(text)
         text.gsub(/<\/?[^>]*>/, "")
@@ -26,13 +30,11 @@ module Jkl
       def remove_short_lines(text, words_on_line = 5)
         text = text.gsub(/\s\s/, "\n")
         str = ""
-        # remove short lines - ususally just navigation
-        text.split("\n").each do |l|
-          str << l unless l.count(" ") < words_on_line
+        text.split("\n").each do |line|
+          str << "#{line}\n" unless line.count(" ") < words_on_line
         end
         str
       end
-      
     end
   end
 end

data/test/unit/jkl_test.rb

@@ -2,13 +2,12 @@ require "test/unit"
 require "shoulda"
 require "webmock/test_unit"
 require "yaml"
-
 require_relative "../../lib/jkl"
 
 class JklTest < Test::Unit::TestCase
   include WebMock::API
 
-  context "for documents, plain text and tags" do
+  context "Jkl: When handling documents, plain text and tags" do
     setup do
       @url = "http://www.bbc.co.uk"
       response = File.read('test/fixtures/bbc_story.html')
@@ -18,27 +17,27 @@ class JklTest < Test::Unit::TestCase
           :headers => {'Content-Type' => 'text/html'})
     end
     
-    should "Get a document from a URL" do
+    should "get a document from a URL" do
       doc = Jkl::get(@url)
       assert_not_nil doc
     end
   
-    should "Get the plain text version of a document" do
+    should "get the plain text version of a document" do
       document = Jkl::get(@url)
       text = Jkl::Text::plain_text(document,2)
-      assert_equal 8884, text.length
+      assert_equal 9009, text.length
     end
   
-    should "Get the keywords from a document" do
+    should "get the keywords from a document" do
       document = Jkl::get(@url)
       text = Jkl::Text::plain_text(document,2)
-      tags = Jkl::Extraction::tags(calais_key, text)
+      tags = Jkl::tags(text)
       assert ! tags.empty?
     end
   end
   
-  context "for RSS" do
-    should "Get links from a feed" do
+  context "Jkl: When handling RSS" do
+    should "get links from a feed" do
       feed = "http://feeds.bbci.co.uk/news/rss.xml"
       response = File.read('test/fixtures/topix_rss.xml')
       stub_request(:get, "http://feeds.bbci.co.uk/news/rss.xml").

data/test/unit/text_cleaning_test.rb

@@ -13,7 +13,7 @@ the cat sat on the mat
 a short line
 HTML
       result = Jkl::Text::remove_short_lines input
-      assert result == "the cat sat on the mat"
+      assert result == "the cat sat on the mat\n"
     end
     
     should "Remove shorter lines" do
@@ -23,7 +23,8 @@ the cat sat on the slightly fluffy, yet worn and homely mat
 a short line
 HTML
       result = Jkl::Text::remove_short_lines(input, 8)
-      assert result == "the cat sat on the slightly fluffy, yet worn and homely mat"
+      expected = "the cat sat on the slightly fluffy, yet worn and homely mat\n"
+      assert_equal expected, result
     end
     
     should "Remove script tags" do
@@ -35,7 +36,7 @@ function nofunction(){var bob;}
 a short line
 HTML
       result = Jkl::Text::remove_short_lines input
-      assert result == "the cat sat on the mat"
+      assert result == "the cat sat on the mat\n"
     end
     
     should "Remove html comments" do
@@ -45,7 +46,7 @@ the cat sat on the mat
 a short line
 HTML
       result = Jkl::Text::remove_short_lines input
-      assert result == "the cat sat on the mat"      
+      assert result == "the cat sat on the mat\n"
     end
     
     should "Remove blank lines" do
@@ -55,7 +56,7 @@ the cat sat on the mat
 a short line
 HTML
       result = Jkl::Text::remove_short_lines input
-      assert result == "the cat sat on the mat"
+      assert result == "the cat sat on the mat\n"
     end
     
     should "Strip all tags" do
@@ -63,7 +64,7 @@ HTML
 <p>the cat sat on the mat</p>
 HTML
       result = Jkl::Text::strip_all_tags input
-      assert result == "the cat sat on the mat\n" #TODO fix carriage return
+      assert result == "the cat sat on the mat\n"
     end
   
     should "Clean text" do
@@ -76,7 +77,19 @@ the cat sat on the mat
 some end stuff here
 HTML
       result = Jkl::Text::plain_text(input)
-      assert result == "the cat sat on the mat"
+      assert result == "the cat sat on the mat\n"
+    end
+    
+    should "Remove HTML escaped characters" do
+      input = <<HTML
+Testing, testing, one two three.
+<p><strong>The cat didn&#39;t sit on the mat</strong></p>
+HTML
+      expected = <<EXPECTED
+Testing, testing, one two three.
+The cat didn't sit on the mat
+EXPECTED
+      assert_equal expected, Jkl::Text::plain_text(input, 2)
     end
   end
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 0
-  version: 0.2.0
+  - 1
+  version: 0.2.1
 platform: ruby
 authors: 
 - sshingler
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-28 00:00:00 +00:00
+date: 2011-06-02 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -63,18 +63,18 @@ dependencies:
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: calais
+  name: term-extract
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
         segments: 
         - 0
-        - 0
-        - 11
-        version: 0.0.11
+        - 5
+        - 1
+        version: 0.5.1
   type: :runtime
   version_requirements: *id004
 description: Jakal is a Ruby library for tagging keywords from web pages.
@@ -92,9 +92,17 @@ files:
 - License.txt
 - README.md
 - Rakefile
+- bin/convert_to_should_syntax
+- bin/edit_json.rb
+- bin/minitar
+- bin/nokogiri
+- bin/prettify_json.rb
+- bin/rake
+- bin/rdebug
+- bin/term-extract
+- bin/testrb
 - jkl.gemspec
 - lib/jkl.rb
-- lib/jkl/calais_client.rb
 - lib/jkl/rss_client.rb
 - lib/jkl/text_client.rb
 - test/fixtures/bbc_story.html

data/lib/jkl/calais_client.rb

@@ -1,28 +0,0 @@
-require "calais"
-
-module Jkl
-  module Extraction
-    class << self
-
-      def calais_response(key, text)
-        Calais.process_document(
-            :content => text,
-            :license_id => key
-        )
-      end
-
-      def entities(key,text)
-        calais_response(key, text).entities.map{|e| {e.type => [e.attributes["name"]]}}
-      end
-
-      def tags(key, text)
-        nested_list = {}
-        entities(key,text).each do |a| 
-          nested_list = nested_list.merge!(a){ |key,v1,v2| v1+v2 }
-        end
-        nested_list
-      end
-
-    end
-  end
-end