jakal 0.1.0 → 0.1.1

data/features/http.feature

@@ -24,9 +24,3 @@ Feature: http features
     When I request some RSS
 	  Then I should get a response
     And I should receive some headlines
-
-  Scenario: Work with RSS
-    Given I have some RSS
-	  Then I should receive some headlines
-	  
-	

data/features/step_definitions/calais_steps.rb

@@ -13,7 +13,8 @@ Given /^I have a mock calais response$/ do
 end
 
 When /^I post to calais$/ do
-  @response = Jkl::get_from_calais @text
+  key = YAML::load_file('config/keys.yml')['calais']
+  @response = Jkl::Extraction::get_from_calais(key, @text)
 end
 
 When /^I remove the unwanted items$/ do
@@ -44,7 +45,7 @@ end
 
 When /^I request the nested entities from calais$/ do
   key = YAML::load_file('config/keys.yml')['calais']
-  @response = Jkl::tags key, @text
+  @response = Jkl::Extraction::tags key, @text
 end
 
 Then /^I should receive the entities grouped into categories$/ do

data/features/step_definitions/http_steps.rb

@@ -9,7 +9,7 @@ end
 When /^I request some RSS$/ do
   keyphrase = @keyphrase || "iraq"
   url = "#{YAML::load_file('config/config.yml')['topix']}#{CGI::escape(keyphrase)}"
-  @response = Jkl::get_from_as_xml url
+  @response = Jkl::get_xml_from url
 end
 
 Given /^I have some RSS$/ do
@@ -28,17 +28,16 @@ When /^I request some trends$/ do
   @response = output['trends']
 end
 
-
 Then /^I should get a response$/ do
   @response.should_not == nil
   #puts @response.inspect
 end
 
 Then /^I should receive some headlines$/ do
-  @items = Jkl::get_items_from @response
+  @items = Jkl::Rss::items @response
   @links = []
   @items.each do |item|
-    @links << Jkl::attribute_from(item, :link)
+    @links << Jkl::Rss::attribute_from(item, :link)
   end
   @links.should_not == nil
   @links.length.should > 0

data/lib/jkl.rb

@@ -1,56 +1,8 @@
-require "cgi"
 require "jkl/rest_client.rb"
 require "jkl/rss_client.rb"
 require "jkl/calais_client.rb"
 require "jkl/url_doc_handler.rb"
 
 module Jkl
-  
-  class << self
 
-    def headlines(feed, keyphrase)
-      get_from_as_xml "#{feed}#{keyphrase}"
-    end
-
-    def pages(headlines)
-      items = get_items_from headlines
-      descriptions = ""
-      items.each do |item| 
-        descriptions << attribute_from(item, :description).gsub("<![CDATA[","").gsub("]]>","")
-      end
-      descriptions
-    end
-
-    def descriptions(headlines)
-      items = get_items_from headlines
-      descriptions = []
-      items.each do |item| 
-        descriptions << attribute_from(item, :description).gsub("<![CDATA[","").gsub("]]>","")
-      end
-      descriptions
-    end
-
-    def links(headlines)
-      items = get_items_from headlines
-      links = []
-      items.each do |item| 
-        links << attribute_from(item, :link)
-      end
-      links
-    end
-
-    def tags(key, text)
-      nested_list = {}
-      entities(key,text).each do |a| 
-        nested_list = nested_list.merge!(a){ |key,v1,v2| v1+v2 }
-      end
-      nested_list
-    end
-    
-    def entities(key,text)
-      calais_response(key, text).entities.map{|e| {e.type => [e.attributes["name"]]}}
-    end
-    
-  end
-  
 end

data/lib/jkl/calais_client.rb

@@ -1,72 +1,81 @@
 require "json"
-require "rest_client"
 require "calais"
 
-module Jkl
+require "rest_client"
 
-  class << self
+module Jkl
+  module Extraction
+    class << self
     
-    #using the calais gem
-    def calais_response(key, pages)
-      Calais.process_document(
-          :content => pages,
-          :content_type => :text, 
-          :license_id => key
-      )
-    end
+      #using the calais gem
+      def calais_response(key, pages)
+        Calais.process_document(
+            :content => pages,
+            :content_type => :text, 
+            :license_id => key
+        )
+      end
   
-    def get_from_calais(content)  
-      begin
-        license_id = YAML::load_file('config/keys.yml')['calais']
-        c_uri = URI.parse('http://api.opencalais.com/enlighten/rest/')
-        post_args = { 'licenseID' => license_id, 'content' => content, 
-                    'paramsXML' => paramsXML('application/json') }
-        post_to(c_uri, post_args)
-      rescue Exception => e
-        puts e
+      def tags(key, text)
+        nested_list = {}
+        entities(key,text).each do |a| 
+          nested_list = nested_list.merge!(a){ |key,v1,v2| v1+v2 }
+        end
+        nested_list
+      end
+    
+      def entities(key,text)
+        calais_response(key, text).entities.map{|e| {e.type => [e.attributes["name"]]}}
+      end
+    
+      #not using calais gem, experimenting with json response
+      def get_from_calais(key, content)  
+        post_args = { 
+            "licenseID" => key, 
+            "content" => content, 
+            "paramsXML" => paramsXML("application/json") 
+        }
+        Jkl::post_to(URI.parse("http://api.opencalais.com/enlighten/rest/"), post_args)
       end
-    end
   
-    def get_tag_from_json(response)
-      result = JSON.parse response
-      result.delete_if {|key, value| key == "doc" } # ditching the doc
-      cleaned_result = []
-      result.each do |key,tag| 
-        tag = Jkl::clean_unwanted_items_from_hash tag
-        cleaned_result << tag
-        yield tag if block_given?
+      def get_tag_from_json(response)
+        result = JSON.parse response
+        result.delete_if {|key, value| key == "doc" } # ditching the doc
+        cleaned_result = []
+        result.each do |key,tag| 
+          tag = Jkl::clean_unwanted_items_from_hash tag
+          cleaned_result << tag
+          yield tag if block_given?
+        end
+        cleaned_result
       end
 
-      cleaned_result
-    end
-
-    #jkl doesn't work with these aspects of the calais response, also removing blanks
-    def clean_unwanted_items_from_hash h
-      h.delete_if {|k, v| k == "relevance" }
-      h.delete_if {|k, v| k == "instances" }
-      h.delete_if {|k, v| v == "N/A"}
-      h.delete_if {|k, v| v == []}
-      h.delete_if {|k, v| v == ""}
-      h.delete_if {|k, v| k == "_typeGroup"}
-      h
-    end
+      def clean_unwanted_items_from_hash h
+        h.delete_if {|k, v| k == "relevance" }
+        h.delete_if {|k, v| k == "instances" }
+        h.delete_if {|k, v| v == "N/A"}
+        h.delete_if {|k, v| v == []}
+        h.delete_if {|k, v| v == ""}
+        h.delete_if {|k, v| k == "_typeGroup"}
+        h
+      end
   
-    private
+      private
 
-    def paramsXML(format)
-     <<-paramsXML; 
-      <c:params xmlns:c="http://s.opencalais.com/1/pred/"
-             xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
-             <c:processingDirectives
-             c:contentType="text/txt"
-             c:outputFormat="#{format}">
-             </c:processingDirectives>  
-             <c:userDirectives />
-             <c:externalMetadata />
-             </c:params>
-      paramsXML
-    end
+      def paramsXML(format)
+       <<-paramsXML; 
+        <c:params xmlns:c="http://s.opencalais.com/1/pred/"
+               xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+               <c:processingDirectives
+               c:contentType="text/txt"
+               c:outputFormat="#{format}">
+               </c:processingDirectives>  
+               <c:userDirectives />
+               <c:externalMetadata />
+               </c:params>
+        paramsXML
+      end
 
+    end
   end
-
 end

data/lib/jkl/rest_client.rb

@@ -2,7 +2,6 @@ require 'net/http'
 require 'hpricot'
 
 module Jkl
-  
   class << self
     
     def post_to(uri, post_args)
@@ -31,10 +30,9 @@ module Jkl
       end
     end
   
-    def get_from_as_xml(uri)
+    def get_xml_from(uri)
       Hpricot.XML get_from uri
     end
   
   end
-
-end
+end

data/lib/jkl/rss_client.rb

@@ -1,19 +1,27 @@
 require 'hpricot'
 
 module Jkl
-  
-  class << self
+  module Rss
+    class << self
 
-    def get_items_from(rssdoc)
-      items = []
-      (rssdoc/:item).each { |rssitem| items.push rssitem } unless rssdoc==nil
-      items
-    end
-  
-    def attribute_from(item, name)
-      (item/name).inner_html
+      def items(rss_doc)
+        (rss_doc/:item)
+      end
+
+      def links(items)
+        items.map{|item| attribute_from(item,:link)}
+      end
+
+      def descriptions(items)
+        items.map do |item| 
+          attribute_from(item, :description).gsub("<![CDATA[","").gsub("]]>","")
+        end
+      end
+
+      def attribute_from(item, name)
+        (item/name).inner_html
+      end
+      
     end
-  
   end
-  
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: jakal
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors: 
 - sshingler