jakal 0.0.9 → 0.1.0

data/features/calais.feature

@@ -21,7 +21,7 @@ Feature: Calais-Specific features
 	Scenario: Get nested tags from calais
 	  Given I have some simple text
     When I request the nested entities from calais
-    Then I should get a decent response
+    Then I should receive the entities grouped into categories
       
   Scenario: Clean up blank items from a calais response
   	Given I have a mock calais response

data/features/step_definitions/calais_steps.rb

@@ -44,9 +44,9 @@ end
 
 When /^I request the nested entities from calais$/ do
   key = YAML::load_file('config/keys.yml')['calais']
-  @response = Jkl::get_calais_metadata key, @text
+  @response = Jkl::tags key, @text
 end
 
-Then /^I should get a decent response$/ do
+Then /^I should receive the entities grouped into categories$/ do
   @response.eql?({"Person"=>["Barack Obama", "Hillary Clinton"], "Position"=>["Secretary of State"]}).should == true
 end

data/lib/jkl.rb

@@ -6,43 +6,51 @@ require "jkl/url_doc_handler.rb"
 
 module Jkl
   
-  def self.headlines(feed, keyphrase)
-    get_from_as_xml "#{feed}#{keyphrase}"
-  end
+  class << self
 
-  def self.pages(headlines)
-    items = get_items_from headlines
-    descriptions = ""
-    items.each do |item| 
-      descriptions << attribute_from(item, :description).gsub("<![CDATA[","").gsub("]]>","")
+    def headlines(feed, keyphrase)
+      get_from_as_xml "#{feed}#{keyphrase}"
     end
-    descriptions
-  end
-  
-  def self.descriptions(headlines)
-    items = get_items_from headlines
-    descriptions = []
-    items.each do |item| 
-      descriptions << attribute_from(item, :description).gsub("<![CDATA[","").gsub("]]>","")
+
+    def pages(headlines)
+      items = get_items_from headlines
+      descriptions = ""
+      items.each do |item| 
+        descriptions << attribute_from(item, :description).gsub("<![CDATA[","").gsub("]]>","")
+      end
+      descriptions
     end
-    descriptions
-  end
-  
-  def self.links(headlines)
-    items = get_items_from headlines
-    links = []
-    items.each do |item| 
-      links << attribute_from(item, :link)
+
+    def descriptions(headlines)
+      items = get_items_from headlines
+      descriptions = []
+      items.each do |item| 
+        descriptions << attribute_from(item, :description).gsub("<![CDATA[","").gsub("]]>","")
+      end
+      descriptions
+    end
+
+    def links(headlines)
+      items = get_items_from headlines
+      links = []
+      items.each do |item| 
+        links << attribute_from(item, :link)
+      end
+      links
     end
-    links
-  end
 
-  def self.tags(key, pages)
-    Calais.process_document(
-        :content => pages,
-        :content_type => :text, 
-        :license_id => key
-    )
+    def tags(key, text)
+      nested_list = {}
+      entities(key,text).each do |a| 
+        nested_list = nested_list.merge!(a){ |key,v1,v2| v1+v2 }
+      end
+      nested_list
+    end
+    
+    def entities(key,text)
+      calais_response(key, text).entities.map{|e| {e.type => [e.attributes["name"]]}}
+    end
+    
   end
   
 end

data/lib/jkl/calais_client.rb

@@ -4,62 +4,69 @@ require "calais"
 
 module Jkl
 
-  def self.get_from_calais(content)  
-    begin
-      license_id = YAML::load_file('config/keys.yml')['calais']
-      c_uri = URI.parse('http://api.opencalais.com/enlighten/rest/')
-      post_args = { 'licenseID' => license_id, 'content' => content, 
-                  'paramsXML' => paramsXML('application/json') }
-      post_to(c_uri, post_args)
-    rescue Exception => e
-      puts e
+  class << self
+    
+    #using the calais gem
+    def calais_response(key, pages)
+      Calais.process_document(
+          :content => pages,
+          :content_type => :text, 
+          :license_id => key
+      )
     end
-  end
   
-  def self.get_tag_from_json(response)
-    result = JSON.parse response
-    result.delete_if {|key, value| key == "doc" } # ditching the doc
-    cleaned_result = []
-    result.each do |key,tag| 
-      tag = Jkl::clean_unwanted_items_from_hash tag
-      cleaned_result << tag
-      yield tag if block_given?
+    def get_from_calais(content)  
+      begin
+        license_id = YAML::load_file('config/keys.yml')['calais']
+        c_uri = URI.parse('http://api.opencalais.com/enlighten/rest/')
+        post_args = { 'licenseID' => license_id, 'content' => content, 
+                    'paramsXML' => paramsXML('application/json') }
+        post_to(c_uri, post_args)
+      rescue Exception => e
+        puts e
+      end
     end
+  
+    def get_tag_from_json(response)
+      result = JSON.parse response
+      result.delete_if {|key, value| key == "doc" } # ditching the doc
+      cleaned_result = []
+      result.each do |key,tag| 
+        tag = Jkl::clean_unwanted_items_from_hash tag
+        cleaned_result << tag
+        yield tag if block_given?
+      end
 
-    cleaned_result
-  end
-
-  def self.get_calais_metadata(key, text)
-    name_value_pairs = Jkl::tags(key, text).entities.map{|e| {e.type => [e.attributes["name"]]}}
-    nested_list = {}
-    name_value_pairs.each { |a| nested_list = nested_list.merge!(a){ |key,v1,v2| v1+v2 }}
-    nested_list
-  end
+      cleaned_result
+    end
 
-  #jkl doesn't work with these aspects of the calais response, also removing blanks
-  def self.clean_unwanted_items_from_hash h
-    h.delete_if {|k, v| k == "relevance" }
-    h.delete_if {|k, v| k == "instances" }
-    h.delete_if {|k, v| v == "N/A"}
-    h.delete_if {|k, v| v == []}
-    h.delete_if {|k, v| v == ""}
-    h.delete_if {|k, v| k == "_typeGroup"}
-    h
-  end
+    #jkl doesn't work with these aspects of the calais response, also removing blanks
+    def clean_unwanted_items_from_hash h
+      h.delete_if {|k, v| k == "relevance" }
+      h.delete_if {|k, v| k == "instances" }
+      h.delete_if {|k, v| v == "N/A"}
+      h.delete_if {|k, v| v == []}
+      h.delete_if {|k, v| v == ""}
+      h.delete_if {|k, v| k == "_typeGroup"}
+      h
+    end
   
-  private
+    private
+
+    def paramsXML(format)
+     <<-paramsXML; 
+      <c:params xmlns:c="http://s.opencalais.com/1/pred/"
+             xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+             <c:processingDirectives
+             c:contentType="text/txt"
+             c:outputFormat="#{format}">
+             </c:processingDirectives>  
+             <c:userDirectives />
+             <c:externalMetadata />
+             </c:params>
+      paramsXML
+    end
 
-  def self.paramsXML(format)
-   <<-paramsXML; 
-    <c:params xmlns:c="http://s.opencalais.com/1/pred/"
-           xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
-           <c:processingDirectives
-           c:contentType="text/txt"
-           c:outputFormat="#{format}">
-           </c:processingDirectives>  
-           <c:userDirectives />
-           <c:externalMetadata />
-           </c:params>
-    paramsXML
   end
+
 end

data/lib/jkl/rest_client.rb

@@ -3,34 +3,38 @@ require 'hpricot'
 
 module Jkl
   
-  def self.post_to(uri, post_args)
-    begin
-      resp, data = Net::HTTP.post_form(uri, post_args)
-      data
-    rescue  URI::InvalidURIError => e
-      puts("WARN: Invalid URI: #{e}")
-    rescue SocketError => e
-      puts("WARN: Could not connect: #{e}")
-    rescue Errno::ECONNREFUSED  => e
-      puts("WARN: Connection refused: #{e}")
+  class << self
+    
+    def post_to(uri, post_args)
+      begin
+        resp, data = Net::HTTP.post_form(uri, post_args)
+        data
+      rescue  URI::InvalidURIError => e
+        puts("WARN: Invalid URI: #{e}")
+      rescue SocketError => e
+        puts("WARN: Could not connect: #{e}")
+      rescue Errno::ECONNREFUSED  => e
+        puts("WARN: Connection refused: #{e}")
+      end
     end
-  end
   
-  def self.get_from(uri)
-    begin
-      res = Net::HTTP.get_response(URI.parse(uri))
-      res.body
-    rescue  URI::InvalidURIError => e
-      puts("WARN: Invalid URI: #{e}")
-    rescue SocketError => e
-      puts("WARN: Could not connect: #{e}")
-    rescue Errno::ECONNREFUSED  => e
-      puts("WARN: Connection refused: #{e}")
+    def get_from(uri)
+      begin
+        res = Net::HTTP.get_response(URI.parse(uri))
+        res.body
+      rescue  URI::InvalidURIError => e
+        puts("WARN: Invalid URI: #{e}")
+      rescue SocketError => e
+        puts("WARN: Could not connect: #{e}")
+      rescue Errno::ECONNREFUSED  => e
+        puts("WARN: Connection refused: #{e}")
+      end
     end
-  end
   
-  def self.get_from_as_xml(uri)
-    Hpricot.XML get_from uri
-  end
+    def get_from_as_xml(uri)
+      Hpricot.XML get_from uri
+    end
   
+  end
+
 end

data/lib/jkl/rss_client.rb

@@ -2,14 +2,18 @@ require 'hpricot'
 
 module Jkl
   
-  def self.get_items_from(rssdoc)
-    items = []
-    (rssdoc/:item).each { |rssitem| items.push rssitem } unless rssdoc==nil
-    items
-  end
+  class << self
+
+    def get_items_from(rssdoc)
+      items = []
+      (rssdoc/:item).each { |rssitem| items.push rssitem } unless rssdoc==nil
+      items
+    end
+  
+    def attribute_from(item, name)
+      (item/name).inner_html
+    end
   
-  def self.attribute_from(item, name)
-    (item/name).inner_html
   end
   
 end

data/lib/jkl/url_doc_handler.rb

@@ -3,29 +3,33 @@ require 'rest_client'
 
 module Jkl
   
-  def self.sanitize(text)
-    str = ""
-    text = text.to_s.gsub(/((<[\s\/]*script\b[^>]*>)([^>]*)(<\/script>))/i,"") #remove script tags - with contents
-    text.to_s.gsub(/<\/?[^>]*>/, "").split("\r").each do |l| # remove all tags
-      l = l.gsub(/^[ \t]/,"") #remove tabs
-      l = l.gsub(/^[ \s]/,"")
-      l.split("\n").each do |l|
-        str << l unless l.count(" ") < 5 # remove short lines - ususally just navigation
+  class << self
+    
+    def sanitize(text)
+      str = ""
+      text = text.to_s.gsub(/((<[\s\/]*script\b[^>]*>)([^>]*)(<\/script>))/i,"") #remove script tags - with contents
+      text.to_s.gsub(/<\/?[^>]*>/, "").split("\r").each do |l| # remove all tags
+        l = l.gsub(/^[ \t]/,"") #remove tabs
+        l = l.gsub(/^[ \s]/,"")
+        l.split("\n").each do |l|
+          str << l unless l.count(" ") < 5 # remove short lines - ususally just navigation
+        end
       end
+      str
     end
-    str
-  end
-  
-  def self.from_doc(response)
-    begin
-      Hpricot(response)
-    rescue  URI::InvalidURIError => e
-      puts("WARN: Problem with getting a connection: #{e}")
-    rescue SocketError => e
-      puts("WARN: Could not connect to feed: #{e}")
-    rescue Errno::ECONNREFUSED  => e
-      puts("WARN: Connection refused: #{e}")
+
+    def from_doc(response)
+      begin
+        Hpricot(response)
+      rescue  URI::InvalidURIError => e
+        puts("WARN: Problem with getting a connection: #{e}")
+      rescue SocketError => e
+        puts("WARN: Could not connect to feed: #{e}")
+      rescue Errno::ECONNREFUSED  => e
+        puts("WARN: Connection refused: #{e}")
+      end
     end
+
   end
   
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: jakal
 version: !ruby/object:Gem::Version 
-  version: 0.0.9
+  version: 0.1.0
 platform: ruby
 authors: 
 - sshingler
@@ -13,7 +13,7 @@ date: 2009-08-27 00:00:00 +01:00
 default_executable: 
 dependencies: []
 
-description: Jakal is a Ruby library for dealing with information overload.
+description: Jakal is a Ruby library which contains some utilies for connecting to internet based APIs.
 email: "'shingler@gmail.com'"
 executables: []
 
@@ -30,7 +30,6 @@ files:
 - lib/jkl/url_doc_handler.rb
 - features/calais.feature
 - features/http.feature
-- features/processing.feature
 - features/sanitize-text.feature
 - features/mocks/bbc_story.html
 - features/mocks/calais.json
@@ -38,7 +37,6 @@ files:
 - features/mocks/twitter.json
 - features/step_definitions/calais_steps.rb
 - features/step_definitions/http_steps.rb
-- features/step_definitions/processing_steps.rb
 - features/step_definitions/require_steps.rb
 - features/step_definitions/sanitize-text_steps.rb
 - features/step_definitions/twitter_steps.rb
@@ -73,6 +71,6 @@ rubyforge_project:
 rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
-summary: Jakal is a Ruby library for dealing with information overload.
+summary: Jakal is a Ruby library which contains some utilies for connecting to internet based APIs.
 test_files: []
 

data/features/processing.feature

@@ -1,16 +0,0 @@
-Feature: Processing features
-  In order to integrate our apps
-  As a developer
-  I want to make some requests and inspect some responses
-	
-  @connection_needed	 
-  Scenario: end to end flow, keyphrase to tags
-	Given I have a keyphrase 'london restaurants'
-	When I request some RSS
-	Then I should receive some headlines
-	And I should be able to get the copy from the first headline
-	When I post to calais
-	Then I should receive some tags
-	And I should be able to persist these tags
-	When I generate a view of the recent keyword results
-	Then I should see a network graph

data/features/step_definitions/processing_steps.rb

@@ -1,30 +0,0 @@
-
-############### pending steps below ################
-
-
-When /^I request tags for the first story$/ do
-  r = Jkl::get_from_calais @story
-  Jkl::get_tag_from_json(get_from_calais(@story)) do |tag|
-    tag.each{|k,v| puts "#{k} : #{v}"}
-  end
-end
-
-When /^I request stories from Topix$/ do 
-  search_term = 'london'
-  url = "#{YAML::load_file('config/config.yml')['topix']}#{search_term}"
-  @response = Jkl::get_from_as_xml url
-end
-
-When /^I get some news stories from the first keyword$/ do
-  search_term = @trend['name'].gsub('#','') #removing hash from start of trend name
-  search_term = 'london'
-  url = "#{YAML::load_file('config/config.yml')['topix']}#{search_term}"
-  rss_response = Jkl::get_from_as_xml url
-  items = Jkl::get_items_from rss_response
-  links = []
-  items.each do |item|
-    links << attribute_from(item, :link)
-  end
-  @story = Jkl::sanitize Jkl::from_doc Jkl::get_from links[0]
-end
-