whitepaper 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in whitepaper.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 wilkie (Dave Wilkinson II)
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,107 @@
+# Whitepaper
+
+This gem will perform a whitepaper lookup on major scholarly databases. It's purpose is to easily find
+related papers and organize your paper collection. With this application, you can easily download pdfs
+or use it as a library to automatically assign metadata.
+
+Currently, CiteSeerX is the only database it uses along with a google pdf search if CiteSeerX does not
+list a pdf to download.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'whitepaper'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install whitepaper
+
+## Usage
+
+Display usage:
+
+    whitepaper -h
+
+Finding article metadata by title keyword search:
+
+    whitepaper -t "corey multicore"
+
+With no other options, this will print out all information about the paper.
+
+If you want to limit what the program prints, then add one or more output flags:
+
+Printing the article's proper title:
+
+    whitepaper -n -t "xomb multicore"
+
+Printing the article's list of authors:
+
+    whitepaper -a -t "ubiquitous computing"
+
+Printing the article's pdf url:
+
+    whitepaper -p -t "exokernel"
+
+Finally, you can simply have the app download an article and place it in the
+current directory. It will name the file as closely to the title as it can.
+
+Download a pdf by any means necessary by title keyword search:
+
+    whitepaper -d -t "The Design and Implementation of a Log-Structured File System"
+
+### Programmable
+
+To get paper metadata, add whitepaper to your Gemfile:
+
+    gem 'whitepaper'
+
+And require it if necessary: (Your project may auto require libraries in your Gemfile)
+
+    require 'whitepaper'
+
+Invoke with this simple command to look up a paper with the given terms in the title:
+
+    paper = Whitepaper.find_by_title("hierarchial filesystems are dead")
+
+This will give you back a Whitepaper::Paper object! To get a pdf url, just go:
+
+    paper.pdf_urls.first unless paper.pdf_urls.empty?
+
+As you can see, you can get a list of pdf links, so you can try each until you find one
+that actually exists, or as a mirror if the server is down. If postscript is your thing, then check
+out ps_urls and follow the same steps.
+
+To get other metadatas, just follow one of the following lines of code:
+
+    title       = paper.title
+    authors     = paper.authors
+    description = paper.description
+    year        = paper.year
+    conference  = paper.conference
+    keywords    = paper.keywords
+
+A field that does not have a value will be a blank string or an empty array. Use the
+empty? method on what is returned in either case to check
+for a field that is, well, empty.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+Please refer to the LICENSE file in this repo for distribution information.
+Whitespace uses an unaltered MIT license, which is a permissive open source
+license. If this is unacceptable for you, please defer to the copyright holder.
+
+### TODO
+
+1. Add new output options (JSON, YAML, etc) for better metadata usage by other programs.
+2. Add new engines (Google Scholar, IEEE, ACM, etc)

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/whitepaper

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname(__FILE__) + '/../lib'
+
+require 'whitepaper'
+require 'whitepaper/cli'
+
+Whitepaper::CLI.run

data/lib/whitepaper/cli.rb

@@ -0,0 +1,99 @@
+require 'optparse'
+
+require 'whitepaper'
+
+module Whitepaper
+  class CLI
+    BANNER = <<-USAGE
+    USAGE
+
+    class << self
+      def parse_options
+        options = {}
+        @opts = OptionParser.new do |opts|
+          opts.banner = BANNER.gsub(/^    /, '')
+
+          opts.separator ''
+          opts.separator 'Options:'
+
+          opts.on('-h', '--help', 'Display this help') do
+            puts opts
+            exit 0
+          end
+
+          opts.on('-t', '--by-title KEYWORDS', 'Display the data for the paper with the given KEYWORDS in title') do |title|
+            options[:by_title] = title
+          end
+          
+          opts.on('-d', '--download', 'Downloads a pdf of the paper of the paper found') do 
+            options[:download] = true
+          end
+
+          opts.on('-p', '--pdf', 'Display a link to the pdf of the paper found') do
+            options[:print_pdf_url] = true
+          end
+
+          opts.on('-n', '--name', 'Display the title of the paper found') do
+            options[:print_title] = true
+          end
+
+          opts.on('-a', '--authors', 'Display the authors of the paper found') do
+            options[:print_authors] = true
+          end
+
+        end
+
+        @opts.parse!
+
+        if options[:by_title]
+          paper = Whitepaper.find_by_title(options[:by_title])
+
+          if options[:print_title]
+            puts paper.title
+          end
+
+          if options[:print_authors]
+            puts paper.authors
+          end
+
+          if options[:print_pdf_url]
+            unless paper.pdf_urls.empty?
+              puts paper.pdf_urls.first
+            end
+          end
+
+          unless options[:print_title] or
+                 options[:print_authors] or
+                 options[:print_pdf_url]
+            puts paper
+          end
+
+          if options[:download] and not paper.pdf_urls.empty?
+            puts "Downloading: " + paper.pdf_urls.first
+            paper.download
+          end
+        else
+          puts opts
+        end
+      end
+
+      def CLI.run
+        begin
+          parse_options
+        rescue OptionParser::InvalidOption => e
+          warn e
+          exit -1
+        end
+
+        def fail
+          puts @opts
+          exit -1
+        end
+
+        # Default
+        puts BANNER
+        exit 0
+      end
+    end
+  end
+end

data/lib/whitepaper/engine/citeseerx.rb

@@ -0,0 +1,87 @@
+require 'mechanize'
+
+require 'whitepaper/paper'
+
+module Whitepaper
+  module Engine
+    module CiteSeerX
+      DOMAIN = "http://citeseerx.ist.psu.edu"
+      SEARCH_BY_TITLE_URL = "search?q=title%3A{title}&t=doc&sort=cite"
+
+      class << self
+        def find_by_title_url(title)
+          "#{DOMAIN}/#{SEARCH_BY_TITLE_URL.gsub(/\{title\}/, title)}"
+        end
+
+        def find_by_title(title)
+          @agent = Mechanize.new
+          page = @agent.get "#{find_by_title_url(title)}"
+
+          # get the first link
+          paper = page.search '//div[@id="result_list"]/div[@class="result"]/h3/a'
+
+          paper_link = "#{DOMAIN}#{paper.first.attribute("href")}"
+
+          retrieve_details paper_link
+        end
+
+        def retrieve_details(url)
+          @agent = Mechanize.new
+
+          page = @agent.get url
+
+          def get_meta(name, page)
+            meta = page.search "//meta[@name=\"#{name}\"]"
+            if meta.nil? or meta.first.nil?
+              return ""
+            end
+            meta.first.attribute "content"
+          end
+
+          description = get_meta("description", page)
+          keywords_raw = get_meta("keywords", page)
+          title = get_meta("citation_title", page)
+          authors_raw = get_meta("citation_authors", page)
+          year = get_meta("citation_year", page)
+          conference = get_meta("citation_conference", page)
+
+          authors = authors_raw.to_s.split(',').map(&:strip)
+          keywords = keywords_raw.to_s.split(',').map(&:strip)
+
+          links = []
+          ps_links = []
+
+          link_url = page.search '//ul[@id="clinks"]/li/a'
+          link_url.each do |l|
+            url = "#{DOMAIN}#{l.attribute("href").to_s}"
+            if url.end_with? "pdf"
+              links << url
+            end
+            if url.end_with? "ps"
+              ps_links << url
+            end
+          end
+
+          link_url = page.search '//ul[@id="dlinks"]/li/a'
+          link_url.each do |l|
+            url = l.attribute("href").to_s
+            if url.end_with? "pdf"
+              links << url
+            end
+            if url.end_with? "ps"
+              ps_links << url
+            end
+          end
+
+          Paper.new title, authors, {:description => description,
+                                     :keywords    => keywords,
+                                     :year        => year,
+                                     :conference  => conference,
+                                     :pdf_urls    => links,
+                                     :ps_urls     => ps_links}
+        end
+      end
+    end
+  end
+end
+

data/lib/whitepaper/engine/google.rb

@@ -0,0 +1,45 @@
+require 'mechanize'
+
+require 'whitepaper/paper'
+
+module Whitepaper
+  module Engine
+    module Google
+      class << self
+        def find_by_title(title)
+          @agent = Mechanize.new
+
+          page = @agent.get "https://www.google.com/search?q=#{URI::encode(title)}+filetype%3Apdf"
+
+          results = page.search '//h3[@class="r"]'
+
+          urls = results.map do |r|
+            a = r.search './a'
+
+            # sanitize
+            url = a.attribute "href"
+
+            url = url.to_s.match(/\/url\?q=([^&]+)&/)[1]
+
+            title = a.first.content
+
+            author = r.search '../div[@class="s"]/span[@class="f"]'
+
+            authors = author.map do |e|
+              e.content.to_s
+            end
+
+            {:url => url, :title => title, :authors => authors}
+          end
+
+          if urls.length > 0
+            Paper.new(urls[0][:title], urls[0][:authors], {:pdf_urls => [urls[0][:url]]})
+          else
+            nil
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/whitepaper/finder.rb

@@ -0,0 +1,4 @@
+module Whitepaper
+  class Finder
+  end
+end

data/lib/whitepaper/paper.rb

@@ -0,0 +1,65 @@
+module Whitepaper
+  class Paper
+    attr_reader :title
+    attr_reader :authors
+    attr_reader :description
+    attr_reader :keywords
+    attr_reader :year
+    attr_reader :conference
+
+    attr_reader :pdf_urls
+    attr_reader :ps_urls
+
+    def initialize(title, authors, options = {})
+      @title = title
+      @authors = authors
+      @description = options[:description] || ""
+      @keywords = options[:keywords] || []
+      @year = options[:year] || ""
+      @conference = options[:conference] || ""
+
+      @pdf_urls = options[:pdf_urls] || []
+      @ps_urls = options[:ps_urls] || []
+    end
+
+    def download(filename = nil)
+      if filename.nil?
+        filename = title.to_s
+      end
+      escaped_filename = filename.gsub(/[\t:\?\<\>\*\"\\\/]/, "") + ".pdf"
+
+      f = open(escaped_filename, "w+")
+
+      if pdf_urls.empty?
+        return false
+      end
+
+      uri = URI.parse(pdf_urls.first)
+      begin
+        Net::HTTP.start(uri.host, uri.port) do |http|
+          http.request_get(uri.request_uri) do |resp|
+            resp.read_body do |segment|
+              f.write(segment)
+            end
+          end
+        end
+      ensure
+        f.close()
+      end
+
+      true
+    end
+
+    def to_s
+      "Title:       #{@title}\n" +
+      "Authors:     #{@authors}\n" +
+      "Description: #{@description}\n" +
+      "Keywords:    #{@keywords}\n" +
+      "Year:        #{@year}\n" +
+      "Conference:  #{@conference}\n" + 
+
+      "Pdf Available: #{@pdf_urls}\n" +
+      "Ps Available: #{@ps_urls}"
+    end
+  end
+end

data/lib/whitepaper/version.rb

@@ -0,0 +1,3 @@
+module Whitepaper
+  VERSION = "0.0.1"
+end

data/lib/whitepaper.rb

@@ -0,0 +1,56 @@
+require "whitepaper/version"
+
+require 'whitepaper/engine/citeseerx'
+require 'whitepaper/engine/google'
+
+module Whitepaper
+  class << self
+    def find_by_title(title)
+      paper = Engine::CiteSeerX.find_by_title(title)
+
+      if paper.pdf_urls.empty?
+        g = Engine::Google.find_by_title(title)
+
+        paper = Paper.new(paper.title,
+                          paper.authors,
+                          {:description => paper.description,
+                           :keywords => paper.keywords,
+                           :year => paper.year,
+                           :conference => paper.conference,
+                           :pdf_urls => g.pdf_urls,
+                           :ps_urls => paper.ps_urls})
+      end
+
+      paper
+    end
+
+    def find_authors_by_title(title)
+      paper = find_by_title(title)
+
+      if paper
+        paper.authors
+      end
+    end
+
+    def find_title_by_title(title)
+      paper = find_by_title(title)
+
+      if paper
+        paper.title
+      end
+    end
+
+    def find_pdfs_by_title(title)
+      paper = find_by_title(title)
+
+      if paper
+        paper.pdf_urls
+      end
+    end
+
+    def download_pdf_by_title(title)
+      paper = find_by_title(title)
+      paper.download
+    end
+  end
+end

data/whitepaper.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'whitepaper/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "whitepaper"
+  gem.version       = Whitepaper::VERSION
+  gem.authors       = ["wilkie"]
+  gem.email         = ["wilkie05@gmail.com"]
+  gem.description   = %q{Finds metadata on scholarly works and is able to download pdfs of whitepapers.}
+  gem.summary       = %q{Finds whitepaper metadata and pdf download links with a basic keyword query using web-based databases such as Google and CiteSeerX.}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency "nokogiri"
+  gem.add_dependency "mechanize"
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: whitepaper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- wilkie
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-01-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Finds metadata on scholarly works and is able to download pdfs of whitepapers.
+email:
+- wilkie05@gmail.com
+executables:
+- whitepaper
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/whitepaper
+- lib/whitepaper.rb
+- lib/whitepaper/cli.rb
+- lib/whitepaper/engine/citeseerx.rb
+- lib/whitepaper/engine/google.rb
+- lib/whitepaper/finder.rb
+- lib/whitepaper/paper.rb
+- lib/whitepaper/version.rb
+- whitepaper.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Finds whitepaper metadata and pdf download links with a basic keyword query
+  using web-based databases such as Google and CiteSeerX.
+test_files: []