whitepaper 0.0.3 → 0.0.4

data/README.md

@@ -1,11 +1,11 @@
 # Whitepaper
 
-This gem will perform a whitepaper lookup on major scholarly databases. It's purpose is to easily find
+This gem will perform a whitepaper lookup on major scholarly databases. Its purpose is to easily find
 related papers and organize your paper collection. With this application, you can easily download pdfs
 or use it as a library to automatically assign metadata.
 
-Currently, CiteSeerX is the only database it uses along with a google pdf search if CiteSeerX does not
-list a pdf to download.
+Currently, CiteSeerX, ACM and IEEE are the only databases it uses, along with a
+google pdf/ps search to find other pdf or ps links to download.
 
 ## Installation
 
@@ -21,7 +21,10 @@ Or install it yourself as:
 
     $ gem install whitepaper
 
-## Usage
+## Command-line Usage
+
+The command-line tool makes it easy to find metadata and download pdf copies of
+papers found via keyword search. It is really only designed for personal use.
 
 Display usage:
 
@@ -50,7 +53,7 @@ Printing the article's pdf url:
 Finally, you can simply have the app download an article and place it in the
 current directory. It will name the file as closely to the title as it can.
 
-Download a pdf by any means necessary by title keyword search:
+Download a PDF by any means necessary by title keyword search:
 
     whitepaper -d -t "The Design and Implementation of a Log-Structured File System"
 
@@ -66,7 +69,7 @@ And require it if necessary: (Your project may auto require libraries in your Ge
 
 Invoke with this simple command to look up a paper with the given terms in the title:
 
-    paper = Whitepaper.find_by_title("hierarchial filesystems are dead")
+    paper = Whitepaper.find_by_title("hierarchical file systems are dead")
 
 This will give you back a Whitepaper::Paper object! To get a pdf url, just go:
 
@@ -104,4 +107,4 @@ license. If this is unacceptable for you, please defer to the copyright holder.
 ### TODO
 
 1. Add new output options (JSON, YAML, etc) for better metadata usage by other programs.
-2. Add new engines (Google Scholar, IEEE, ACM, etc)
+2. Add new engines (Google Scholar, etc)

data/lib/whitepaper.rb

@@ -1,6 +1,7 @@
 require "whitepaper/version"
 
 require 'whitepaper/engine/acm'
+require 'whitepaper/engine/ieeexplore'
 require 'whitepaper/engine/citeseerx'
 require 'whitepaper/engine/google'
 
@@ -15,22 +16,28 @@ module Whitepaper
     def find_by_title(title)
       paper_csx = Engine::CiteSeerX.find_by_title(title)
       paper_acm = Engine::ACM.find_by_title(title)
-      paper_i3e = Engine::ACM.find_by_title(title)
+      paper_i3e = Engine::IEEEXplore.find_by_title(title)
 
-      paper = [paper_csx, paper_i3e, paper_acm].sort{|a,b| b.score_by_title(title) <=> a.score_by_title(title)}.first
+      papers = []
+      papers << paper_csx if paper_csx
+      papers << paper_acm if paper_acm
+      papers << paper_i3e if paper_i3e
 
-      if paper.pdf_urls.empty?
-        g = Engine::Google.find_by_title(title)
+      paper = papers.sort{|a,b| b.score_by_title(title) <=> a.score_by_title(title)}.first
 
-        paper = Paper.new(paper.title,
-                          paper.authors,
-                          {:description => paper.description,
-                           :keywords => paper.keywords,
-                           :year => paper.year,
-                           :conference => paper.conference,
-                           :pdf_urls => g.pdf_urls,
-                           :ps_urls => paper.ps_urls})
-      end
+      # Gather pdf and ps links across the open internet
+      g = Engine::Google.find_by_title(title)
+
+      return g if paper.nil?
+
+      paper = Paper.new(paper.title,
+                        paper.authors,
+                        {:description => paper.description,
+                         :keywords => paper.keywords,
+                         :year => paper.year,
+                         :conference => paper.conference,
+                         :pdf_urls => paper.pdf_urls.concat(g.pdf_urls),
+                         :ps_urls => paper.ps_urls.concat(g.ps_urls)})
 
       paper
     end

data/lib/whitepaper/cli.rb

@@ -27,8 +27,8 @@ module Whitepaper
           opts.on('-t', '--by-title KEYWORDS', 'Display the data for the paper with the given KEYWORDS in title') do |title|
             options[:by_title] = title
           end
-          
-          opts.on('-d', '--download', 'Downloads a pdf of the paper of the paper found') do 
+
+          opts.on('-d', '--download', 'Downloads a pdf of the paper of the paper found') do
             options[:download] = true
           end
 
@@ -76,7 +76,7 @@ module Whitepaper
             paper.download
           end
         else
-          puts opts
+          puts @opts
         end
       end
 

data/lib/whitepaper/engine/acm.rb

@@ -10,6 +10,7 @@ module Whitepaper
       # The url to use to search by title.
       SEARCH_BY_TITLE_URL2 = "results.cfm?within={title_query}&adv=1&DL=ACM&termzone=Title&allofem={title}"
 
+      # The alternate url to use to search by title.
       SEARCH_BY_TITLE_URL = "results.cfm?query={title}&querydisp={title}&srt=score%20dsc&short=0&coll=DL&dl=GUIDE&source_disp=&source_query=&since_month=&since_year=&before_month=&before_year=&termshow=matchall&range_query="
 
       class << self

data/lib/whitepaper/engine/citeseerx.rb

@@ -26,6 +26,11 @@ module Whitepaper
           # get the first link
           paper = page.search '//div[@id="result_list"]/div[@class="result"]/h3/a'
 
+          if paper.empty?
+            # no results
+            return nil
+          end
+
           paper_link = "#{DOMAIN}#{paper.first.attribute("href")}"
 
           retrieve_details paper_link

data/lib/whitepaper/engine/google.rb

@@ -7,11 +7,11 @@ module Whitepaper
     # This engine simply uses a google filetype:pdf search to find paper information.
     module Google
       class << self
-        # Finds a Whitespace::Paper by looking up a paper with the given title keywords.
-        def find_by_title(title)
+        # Return the url and title of the first result as a hash with keys :url and :title.
+        def find(url)
           @agent = Mechanize.new
 
-          page = @agent.get "https://www.google.com/search?q=#{URI::encode(title)}+filetype%3Apdf"
+          page = @agent.get url 
 
           results = page.search '//h3[@class="r"]'
 
@@ -35,11 +35,61 @@ module Whitepaper
           end
 
           if urls.length > 0
-            Paper.new(urls[0][:title], urls[0][:authors], {:pdf_urls => [urls[0][:url]]})
+            urls.first
           else
             nil
           end
         end
+
+        # Finds a Whitespace::Paper by looking up a paper with the given title keywords.
+        def find_by_title(title)
+          pdf = find("https://www.google.com/search?q=#{URI::encode(title)}+filetype%3Apdf")
+          ps = find("https://www.google.com/search?q=#{URI::encode(title)}+filetype%3Aps")
+
+          pdf_urls = []
+          ps_urls = []
+
+          pdf_score = score(pdf[:title], title)
+          ps_score  = score(ps[:title],  title)
+
+          if pdf and pdf_score >= ps_score
+            pdf_urls << pdf[:url]
+          end
+
+          if ps and ps_score >= pdf_score
+            ps_urls << ps[:url]
+          end
+
+          Paper.new(pdf[:title], [], {:pdf_urls => pdf_urls,
+                                      :ps_urls  => ps_urls})
+        end
+
+        # Get an early score rating
+        #--
+        # TODO: move into own class for Whitepaper::Paper
+        def score(title, keywords)
+          keywords = keywords.split(" ").map(&:strip).map(&:downcase)
+          title_words = title.split(" ").map(&:strip).map(&:downcase)
+
+          score = 1.0
+
+          # found words are worth x10
+          # not found words are worth /2
+
+          keywords.each do |k|
+            if title_words.include? k
+              score *= 10.0
+            end
+          end
+
+          title_words.each do |k|
+            unless keywords.include? k
+              score /= 2.0
+            end
+          end
+
+          score
+        end
       end
     end
   end

data/lib/whitepaper/engine/ieeexplore.rb

@@ -2,8 +2,10 @@ module Whitepaper
   module Engine
     # This engine uses the IEEEXplore database to query metadata about a paper.
     module IEEEXplore
+      # The domain for IEEEXplore.
       DOMAIN = "http://ieeexplore.ieee.org"
 
+      # The url to use to search by title keywords.
       SEARCH_BY_TITLE_URL = "search/searchresult.jsp?reload=true&newsearch=true&queryText={title}&x=60&y=7"
 
       class << self
@@ -56,7 +58,7 @@ module Whitepaper
           links = []
           ps_links = []
 
-          Paper.new title, authors, {:description  => description,
+          Paper.new title, authors, {:description  => "",
                                      :keywords     => keywords,
                                      :year         => year,
                                      :conference   => conference,

data/lib/whitepaper/version.rb

@@ -1,4 +1,4 @@
 module Whitepaper
   # Version number for Whitepaper gem.
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: whitepaper
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-28 00:00:00.000000000 Z
+date: 2013-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -86,7 +86,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Finds whitepaper metadata and pdf download links with a basic keyword query