sshingler-jkl 0.0.2

data/License.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2008,2009 Steven Shingler
+ 
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+ 
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+ 
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/calais_client.rb

@@ -0,0 +1,52 @@
+require 'json'
+require 'lib/rest_client'
+
+module Jkl
+
+LICENSE_ID = YAML::load_file('config/keys.yml')['calais']
+C_URI = URI.parse('http://api.opencalais.com/enlighten/rest/')
+
+  def get_from_calais(content)  
+    post_args = { 'licenseID' => LICENSE_ID, 'content' => content, 
+                  'paramsXML' => paramsXML('application/json') }
+    post_to(C_URI, post_args)
+  end
+  
+  def get_tag_from_json(response)
+    result = JSON.parse response
+    result.delete_if {|key, value| key == "doc" } # ditching the doc
+    cleaned_result = []
+    result.each do |key,tag| 
+      tag = clean_unwanted_items_from_hash tag
+      cleaned_result << tag
+      yield tag if block_given?
+    end
+    cleaned_result
+  end
+
+  #jkl doesn't work with these aspects of the calais response, also removing blanks
+  def clean_unwanted_items_from_hash h
+    h.delete_if {|k, v| k == "relevance" }
+    h.delete_if {|k, v| k == "instances" }
+    h.delete_if {|k,v| v == "N/A"}
+    h.delete_if {|k,v| v == []}
+    h.delete_if {|k,v| v == ""}
+    h
+  end
+  
+  private
+
+  def paramsXML(format)
+   <<-paramsXML; 
+    <c:params xmlns:c="http://s.opencalais.com/1/pred/"
+           xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+           <c:processingDirectives
+           c:contentType="text/txt"
+           c:outputFormat="#{format}">
+           </c:processingDirectives>  
+           <c:userDirectives />
+           <c:externalMetadata />
+           </c:params>
+    paramsXML
+  end
+end

data/lib/jkl_client.rb

@@ -0,0 +1,26 @@
+require 'lib/rest_client.rb'
+require 'lib/rss_client.rb'
+require 'lib/calais_client.rb'
+require 'lib/url_doc_handler.rb'
+
+module Jkl
+  
+  def headlines(keyphrase)
+    get_from_as_xml "#{YAML::load_file('config/config.yml')['topix']}#{CGI::escape(keyphrase)}"
+  end
+
+  def pages(headlines)
+    items = get_items_from headlines
+    descriptions = ""
+    items.each do |item| 
+      descriptions << attribute_from(item, :description).gsub("<![CDATA[",'').gsub("]]>",'')
+    end
+    descriptions
+  end
+
+  def tags(pages)
+    cal_response = get_from_calais(pages)
+    get_tag_from_json(cal_response)
+  end
+  
+end

data/lib/persistence_client.rb

@@ -0,0 +1,20 @@
+require 'couchrest'
+
+module Jkl
+  
+  SERVER = CouchRest.database! YAML::load_file('config/config.yml')['db']
+  
+  def delete_db
+    SERVER.delete! rescue nil
+  end
+  
+  class Trend < CouchRest::ExtendedDocument
+    
+    use_database SERVER
+    property :name
+    view_by :name
+    timestamps!
+  
+  end
+  
+end

data/lib/rest_client.rb

@@ -0,0 +1,36 @@
+require 'net/http' 
+require 'hpricot'
+
+module Jkl
+  
+  def post_to(uri, post_args)
+    begin
+      resp, data = Net::HTTP.post_form(uri, post_args)
+      data
+    rescue  URI::InvalidURIError => e
+      puts("WARN: Invalid URI: #{e}")
+    rescue SocketError => e
+      puts("WARN: Could not connect: #{e}")
+    rescue Errno::ECONNREFUSED  => e
+      puts("WARN: Connection refused: #{e}")
+    end
+  end
+  
+  def get_from(uri)
+    begin
+      res = Net::HTTP.get_response(URI.parse(uri))
+      res.body
+    rescue  URI::InvalidURIError => e
+      puts("WARN: Invalid URI: #{e}")
+    rescue SocketError => e
+      puts("WARN: Could not connect: #{e}")
+    rescue Errno::ECONNREFUSED  => e
+      puts("WARN: Connection refused: #{e}")
+    end
+  end
+  
+  def get_from_as_xml(uri)
+    Hpricot.XML get_from uri
+  end
+  
+end

data/lib/rss_client.rb

@@ -0,0 +1,15 @@
+require 'hpricot'
+
+module Jkl
+  
+  def get_items_from(rssdoc)
+    items = []
+    (rssdoc/:item).each { |rssitem| items.push rssitem } unless rssdoc==nil
+    items
+  end
+  
+  def attribute_from(item, name)
+    (item/name).inner_html
+  end
+  
+end

data/lib/url_doc_handler.rb

@@ -0,0 +1,27 @@
+require 'hpricot'
+require 'rest_client'
+
+module Jkl
+  
+  def sanitize(text)
+    str = ""
+    text.to_s.gsub(/<\/?[^>]*>/, "").split("\r").each do |l| # remove tags
+      l = l.chomp.gsub("\t",'').gsub(/\s{2,}/,'') # remove tabs and larger spaces
+      str << l unless l.count(" ") < 5 # remove short lines - ususally just navigation
+    end
+    str
+  end
+  
+  def from_doc(response)
+    begin
+      Hpricot(response)
+    rescue  URI::InvalidURIError => e
+      puts("WARN: Problem with getting a connection: #{e}")
+    rescue SocketError => e
+      puts("WARN: Could not connect to feed: #{e}")
+    rescue Errno::ECONNREFUSED  => e
+      puts("WARN: Connection refused: #{e}")
+    end
+  end
+  
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification 
+name: sshingler-jkl
+version: !ruby/object:Gem::Version 
+  version: 0.0.2
+platform: ruby
+authors: 
+- steven shingler
+- richard vaughan
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-08-27 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: Jkl is a Ruby library and Sinatra app for dealing with information overload.
+email: "'shingler@gmail.com'"
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+- License.txt
+files: 
+- lib/calais_client.rb
+- lib/jkl_client.rb
+- lib/persistence_client.rb
+- lib/rest_client.rb
+- lib/rss_client.rb
+- lib/url_doc_handler.rb
+- README
+- License.txt
+has_rdoc: false
+homepage: http://github.com/sshingler/jkl
+licenses: 
+post_install_message: 
+rdoc_options: 
+- --inline-source
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 2
+summary: Jkl is a Ruby library and Sinatra app for dealing with information overload.
+test_files: []
+