whitepaper 0.0.2 → 0.0.3

data/lib/whitepaper/engine/acm.rb

@@ -0,0 +1,89 @@
+require 'mechanize'
+
+module Whitepaper
+  module Engine
+    # This engine uses the ACM database to query metadata about a paper.
+    module ACM
+      # The domain to use for ACM.
+      DOMAIN = "https://dl.acm.org"
+
+      # The url to use to search by title.
+      SEARCH_BY_TITLE_URL2 = "results.cfm?within={title_query}&adv=1&DL=ACM&termzone=Title&allofem={title}"
+
+      SEARCH_BY_TITLE_URL = "results.cfm?query={title}&querydisp={title}&srt=score%20dsc&short=0&coll=DL&dl=GUIDE&source_disp=&source_query=&since_month=&since_year=&before_month=&before_year=&termshow=matchall&range_query="
+
+      class << self
+        # Returns a url that will query for the given title keywords.
+        def find_by_title_url(title)
+          "#{DOMAIN}/#{SEARCH_BY_TITLE_URL
+            .gsub(/\{title\}/, title.gsub(/\s/, "+"))
+            .gsub(/\{title_query\}/, "(Title:\"" + title.split(" ").join("\"+or+Title:\"") + "\")")}"
+        end
+
+        # Returns a Whitespace::Paper by searching for the paper with the given title keywords.
+        def find_by_title(title)
+          @agent = Mechanize.new
+
+          # In case cookies are ever necessary to establish:
+          #page = @agent.get("#{DOMAIN}")
+          #search_url = page.search('//form[@name="qiksearch"]').first.attribute("action").to_s
+
+          page = @agent.get(find_by_title_url(title))
+
+          # get the first link
+          paper = page.search '//a[@class="medium-text"]'
+
+          paper_link = "#{DOMAIN}/#{paper.first.attribute("href")}"
+
+          retrieve_details paper_link
+        end
+
+        # Returns a Whitespace::Paper by reading the direct page for a particular paper.
+        def retrieve_details(url)
+          @agent = Mechanize.new
+
+          page = @agent.get url
+
+          get_meta = lambda {|name|
+            meta = page.search "//meta[@name=\"#{name}\"]"
+            if meta.nil? or meta.first.nil?
+              return ""
+            end
+            meta.first.attribute("content").to_s
+          }
+
+          title = get_meta.call("citation_title")
+          authors_raw = get_meta.call("citation_authors")
+          year = get_meta.call("citation_date")
+          year = year[-4..-1] unless year.empty?
+          conference = get_meta.call("citation_conference")
+          publisher = get_meta.call("citation_publisher")
+
+          authors = authors_raw.to_s.split(';').map(&:strip).map do |s|
+            index = s.index(',')
+            if index > 0
+              "#{s[index+2..-1]} #{s[0..index-1]}"
+            else
+              s
+            end
+          end
+
+          links = []
+          ps_links = []
+
+          # get abstract
+          abstract_url = page.content.match(/tab_abstract\.cfm\?.*cftoken\=\d+/)[0]
+          abstract = @agent.get(abstract_url).root.text.to_s.strip
+
+          Paper.new title, authors, {:description  => abstract,
+                                     :keywords     => [],
+                                     :metadata_url => url,
+                                     :year         => year,
+                                     :conference   => conference,
+                                     :pdf_urls     => links,
+                                     :ps_urls      => ps_links}
+        end
+      end
+    end
+  end
+end

data/lib/whitepaper/engine/citeseerx.rb

@@ -42,7 +42,7 @@ module Whitepaper
             if meta.nil? or meta.first.nil?
               return ""
             end
-            meta.first.attribute "content"
+            meta.first.attribute("content").to_s
           }
 
           description = get_meta.call("description")
@@ -60,32 +60,33 @@ module Whitepaper
 
           link_url = page.search '//ul[@id="clinks"]/li/a'
           link_url.each do |l|
-            url = "#{DOMAIN}#{l.attribute("href").to_s}"
-            if url.end_with? "pdf"
-              links << url
+            purl = "#{DOMAIN}#{l.attribute("href").to_s}"
+            if purl.end_with? "pdf"
+              links << purl
             end
-            if url.end_with? "ps"
-              ps_links << url
+            if purl.end_with? "ps"
+              ps_links << purl
             end
           end
 
           link_url = page.search '//ul[@id="dlinks"]/li/a'
           link_url.each do |l|
-            url = l.attribute("href").to_s
-            if url.end_with? "pdf"
-              links << url
+            purl = l.attribute("href").to_s
+            if purl.end_with? "pdf"
+              links << purl
             end
-            if url.end_with? "ps"
-              ps_links << url
+            if purl.end_with? "ps"
+              ps_links << purl
             end
           end
 
-          Paper.new title, authors, {:description => description,
-                                     :keywords    => keywords,
-                                     :year        => year,
-                                     :conference  => conference,
-                                     :pdf_urls    => links,
-                                     :ps_urls     => ps_links}
+          Paper.new title, authors, {:description  => description,
+                                     :keywords     => keywords,
+                                     :year         => year,
+                                     :conference   => conference,
+                                     :metadata_url => url,
+                                     :pdf_urls     => links,
+                                     :ps_urls      => ps_links}
         end
       end
     end

data/lib/whitepaper/engine/ieeexplore.rb

@@ -0,0 +1,70 @@
+module Whitepaper
+  module Engine
+    # This engine uses the IEEEXplore database to query metadata about a paper.
+    module IEEEXplore
+      DOMAIN = "http://ieeexplore.ieee.org"
+
+      SEARCH_BY_TITLE_URL = "search/searchresult.jsp?reload=true&newsearch=true&queryText={title}&x=60&y=7"
+
+      class << self
+        # Returns a url that will query for the given title keywords
+        def find_by_title_url(title)
+          "#{DOMAIN}/#{SEARCH_BY_TITLE_URL.gsub(/\{title\}/, title.gsub(/\s/, "+"))}"
+        end
+
+        # Returns a Whitespace::Paper by searching for the paper with the given title keywords.
+        def find_by_title(title)
+          @agent = Mechanize.new
+          page = @agent.get "#{find_by_title_url(title)}"
+
+          # get the first link
+          paper = page.search '//div[@class="detail"]/h3/a'
+
+          paper_link = "#{DOMAIN}#{paper.first.attribute("href")}"
+
+          retrieve_details paper_link
+        end
+
+        # Returns a Whitespace::Paper by reading the direct page for a particular paper.
+        def retrieve_details(url)
+          @agent = Mechanize.new
+
+          page = @agent.get url
+
+          get_meta = lambda {|name|
+            meta = page.search "//meta[@property=\"#{name}\"]"
+            if meta.nil? or meta.first.nil?
+              return ""
+            end
+            meta.first.attribute("content").to_s
+          }
+
+          keywords_raw = get_meta.call("keywords")
+          title = get_meta.call("citation_title")
+          year = get_meta.call("citation_date")
+          year = year[-4..-1] unless year.empty?
+          conference = get_meta.call("citation_conference")
+
+          authors = []
+          meta = page.search "//meta[@property=\"citation_author\"]"
+          meta.each do |e|
+            authors << e.attribute("content").to_s.strip
+          end
+
+          keywords = keywords_raw.to_s.split(';').map(&:strip)
+
+          links = []
+          ps_links = []
+
+          Paper.new title, authors, {:description  => description,
+                                     :keywords     => keywords,
+                                     :year         => year,
+                                     :conference   => conference,
+                                     :metadata_url => url,
+                                     :pdf_urls     => links,
+                                     :ps_urls      => ps_links}
+        end
+      end
+    end
+  end
+end

data/lib/whitepaper/paper.rb

@@ -19,6 +19,10 @@ module Whitepaper
     # The conference, if any, the paper appeared. Defaults to "".
     attr_reader :conference
 
+    # The link to the resource with the most metadata to use as attribution.
+    # Defaults to "".
+    attr_reader :metadata_url
+
     # A list of urls to pdf copies of the paper. Defaults to [].
     attr_reader :pdf_urls
 
@@ -34,6 +38,7 @@ module Whitepaper
       @keywords = options[:keywords] || []
       @year = options[:year] || ""
       @conference = options[:conference] || ""
+      @metadata_url = options[:metadata_url] || ""
 
       @pdf_urls = options[:pdf_urls] || []
       @ps_urls = options[:ps_urls] || []
@@ -70,6 +75,32 @@ module Whitepaper
       true
     end
 
+    # Gives a score of relevancy to the title keywords given. Higher scores
+    # mean that the keywords are more reflective of the title.
+    def score_by_title(keywords)
+      keywords = keywords.split(" ").map(&:strip).map(&:downcase)
+      title_words = title.split(" ").map(&:strip).map(&:downcase)
+
+      score = 1.0
+
+      # found words are worth x10
+      # not found words are worth /2
+    
+      keywords.each do |k|
+        if title_words.include? k
+          score *= 10.0
+        end
+      end
+
+      title_words.each do |k|
+        unless keywords.include? k
+          score /= 2.0
+        end
+      end
+
+      score
+    end
+
     # Output a simple description of the paper metadata.
     def to_s
       "Title:       #{@title}\n" +
@@ -78,6 +109,7 @@ module Whitepaper
       "Keywords:    #{@keywords}\n" +
       "Year:        #{@year}\n" +
       "Conference:  #{@conference}\n" + 
+      "More info:   #{@metadata_url}\n" +
 
       "Pdf Available: #{@pdf_urls}\n" +
       "Ps Available: #{@ps_urls}"

data/lib/whitepaper/version.rb

@@ -1,4 +1,4 @@
 module Whitepaper
   # Version number for Whitepaper gem.
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/whitepaper.rb

@@ -1,5 +1,6 @@
 require "whitepaper/version"
 
+require 'whitepaper/engine/acm'
 require 'whitepaper/engine/citeseerx'
 require 'whitepaper/engine/google'
 
@@ -12,7 +13,11 @@ module Whitepaper
   class << self
     # Find and return a Whitepaper::Paper by searching for a partial match with the given title.
     def find_by_title(title)
-      paper = Engine::CiteSeerX.find_by_title(title)
+      paper_csx = Engine::CiteSeerX.find_by_title(title)
+      paper_acm = Engine::ACM.find_by_title(title)
+      paper_i3e = Engine::ACM.find_by_title(title)
+
+      paper = [paper_csx, paper_i3e, paper_acm].sort{|a,b| b.score_by_title(title) <=> a.score_by_title(title)}.first
 
       if paper.pdf_urls.empty?
         g = Engine::Google.find_by_title(title)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: whitepaper
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -59,8 +59,10 @@ files:
 - bin/whitepaper
 - lib/whitepaper.rb
 - lib/whitepaper/cli.rb
+- lib/whitepaper/engine/acm.rb
 - lib/whitepaper/engine/citeseerx.rb
 - lib/whitepaper/engine/google.rb
+- lib/whitepaper/engine/ieeexplore.rb
 - lib/whitepaper/paper.rb
 - lib/whitepaper/version.rb
 - whitepaper.gemspec