rubyscholar 0.0.2

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/README.md

@@ -0,0 +1,40 @@
+# Synopsis
+
+Here is a small script to "scrape" your Google Scholar citations and reformat them (the way I need it for my website). 
+Not super flexible - but should be easily customizable.
+
+Some features: 
+
+ * if registered on Crossref, retreives corresponding DOIs and can add altmetric.org links. 
+   If Crossref doesn't think your email is valid, no DOIs will be retreived. 
+ * adds "Cited by N" for popular papers
+
+# How to use:
+
+1. Configure "config.yml"
+   If you want DOI retreival to work (including Altmetrics), you need to be 
+   registered at crossref (its free). 
+2. Run `ruby bin/scrape.rb > mypublications.html`
+3. Thats it. 
+
+
+# Potential for improvement:
+
+ * uses author list as visible on your main Google Scholar page. Sometimes this 
+   means names are chopped in two or just a single author is missing. This could
+   be made smarter. 
+ * flexible output
+ * flexible use of DOIs
+
+# Technologies
+
+Ruby, Nokogiri. Thanks to Google Scholar and Crossref. I hope none of this infringes on anything. 
+
+# Contact
+
+RubyScholar was developed by Yannick Wurm (http://yannick.poulet.org). Pull requests, patches and bug reports are welcome. The source code is available on github. Bug reports and feature requests may also be made there.
+
+# Copyright
+
+RubyScholar � 2013 by Yannick Wurm. Licensed under the MIT license. 
+

data/bin/scrape.rb

@@ -0,0 +1,22 @@
+require_relative '../lib/rubyscholar'
+require 'yaml'
+
+def scrape()
+    config    = YAML.load_file('config.yml')
+    parsed    = RubyScholar::Parser.new(config["url"], 
+                                    config["email"])
+    formatter = RubyScholar::Formatter.new(parsed, 
+                                       config["highlight"], 
+                                       config["pdfs"], 
+                                       config["altmetricDOIs"], 
+                                       config["minCitations"].to_i)
+
+    html = formatter.to_html
+        config["italicize"].each do |term|
+    html.gsub!( term , '<em>' + term + '</em>')
+    end
+
+    f= File.open('scholar.html','w')
+    f.write html
+    f.close
+end

data/config.yml

@@ -0,0 +1,49 @@
+# Google Scholar page (you can choose how you sort it)
+url: "http://scholar.google.com/citations?sortby=pubdate&hl=en&user=k6y0EGsAAAAJ&view_op=list_works"
+
+# Name to highlight 
+highlight: "Y Wurm"
+
+
+# Need an Email address that has been registered with CrossRef to obtain DOIs 
+# using their OpenURL service. 
+# e.g. the following should provide an XML file: 
+# http://www.crossref.org/openurl?redirect=false&pid=YOUR@EMAIL>COM&aulast=Wurm&atitle=Behavioral%20Genomics:%20A,%20Bee,%20C,%20G,%20T
+email: your@email.com
+
+
+# Show "[Cited Nx]" if N > the following number
+minCitations: 5
+
+# Words to italicize (emphasize). These will have "<em>" around them. 
+italicize: 
+ - Solenopsis invicta
+ - Acromyrmex echinatior
+ - de novo
+
+# DOIs of articles for which we should show altmetric.org badges. 
+altmetricDOIs: 
+ - "10.1038/nature11832"
+ - "10.1101/gr.121392.111"
+ - "10.1073/pnas.1009690108"
+ - "10.1073/pnas.1104825108"
+
+# Article titles for which we have urls to PDFs
+pdfs:
+  "A Y-like social chromosome causes alternative colony organization in fire ants" : "/publications/wangwurm2013socialChromosome.pdf"
+  "Duplication and concerted evolution in a master sex determiner under balancing selection" : "/publications/procb2013.pdf"
+  "Comparative genomics of chemosensory protein genes reveals rapid evolution and positive selection in ant-specific duplicates" : "/publications/hdy2012122a.pdf"
+  "The Molecular Clockwork of the Fire Ant Solenopsis invicta" : "/publications/ingram2012-fireAntClockGenes.pdf"
+  "Epigenetics: The Making of Ant Castes" : "/publications/2012CurrBiolAntepigenetics.pdf"
+  "Visualization and quality assessment of de novo genome assemblies" : "/publications/Bioinformatics-2011-Riba-Grognuz-3425-6"
+  "The genomic impact of 100 million years of social evolution in seven ant species" : "/publications/TiG2011.pdf"
+  "Relaxed selection is a precursor to the evolution of phenotypic plasticity" : "/publications/hunt2011phenotypicPlasticity.pdf"
+  "The genome of the leaf-cutting ant Acromyrmex echinatior suggests key adaptations to advanced social life and fungus farming" : "/publications/nygaard2011-acromyrmex-genome.pdf"
+  "Behind the Scenes of an Ant Genome Project" : "/publications/wurm2011antGenomeBehindTheScenes.pdf"
+  "The genome of the fire ant Solenopsis invicta" : "/publications/wurm2011fireAntGenome.pdf"
+  "Odorant Binding Proteins of the Red Imported Fire Ant, Solenopsis invicta: An Example of the Problems Facing the Analysis of Widely Divergent Proteins" : "/publications/gotzek2011obps.pdf"
+  "Parasitoid Wasps: From Natural History to Genomic Studies" : "/publications/wurm2010wasps.pdf"
+  "Changes in reproductive roles are associated with changes in gene expression in fire ant queens" : "/publications/wurm2010fireAntQueenDealationExpression.pdf"
+  "Fourmidable: a database for ant genomics" : "/publications/wurm2009antDatabase.pdf"
+  "Behavioral Genomics: A, Bee, C, G, T" : "/publications/wurm2007bees.pdf"
+  "An annotated cDNA library and microarray for large-scale gene-expression studies in the ant Solenopsis invicta" : "/publications/wang2007fireAntMicroarrays.pdf"

data/lib/rubyscholar.rb

@@ -0,0 +1,141 @@
+require "nokogiri"
+require "open-uri"
+
+class String
+  def clean
+    # removes leading and trailing whitespace, commas
+    self.gsub!(/(^[\s,]+)|([\s,]+$)/, '')
+    return self
+  end
+end
+
+module RubyScholar
+  class Paper < Struct.new(:title, :url, :authors, :journalName, :journalDetails, :year, :citationCount, :citingPapers, :doi)
+  end  
+  
+  class Parser
+    attr_accessor :parsedPapers, :crossRefEmail
+    
+    def initialize(url, crossRefEmail = "")
+      @parsedPapers  = []
+      @crossRefEmail = crossRefEmail # if nil doesn't return any DOI
+      parse(url)
+    end
+
+    def parse(url)
+      papers = Nokogiri::HTML(open(url)).css(".cit-table .item")
+      STDOUT << "Found #{papers.length} papers.\n"
+      papers.each do |paper|
+        paperDetails   = paper.css("#col-title")
+        title          = paperDetails[0].children[0].content.clean
+        googleUrl      = paperDetails[0].children[0].attribute('href')
+        authors        = paperDetails[0].children[2].content.clean
+        authors.gsub!("...", "et al")
+
+        journal        = paperDetails[0].children[4].content
+        journalName    = journal.split(/,|\d/).first.clean
+        journalDetails = journal.gsub(journalName, '').clean
+
+        year           = paper.css("#col-year").text # is the last thing we get
+
+        #citations
+        citeInfo      = paper.css(".cit-dark-link")
+        citationCount = citeInfo.text
+        citationUrl   = citationCount.empty?  ? nil : citeInfo.attribute('href').to_s 
+
+        # get DOI: needs last name of first author, no funny chars
+        lastNameFirstAuthor = ((authors.split(',').first ).split(' ').last ).gsub(/[^A-Za-z\-]/, '')
+        doi                 = getDoi( lastNameFirstAuthor, title, @crossRefEmail)
+
+        @parsedPapers.push(Paper.new( title, googleUrl, authors, journalName, journalDetails, year, citationCount, citationUrl, doi))
+      end
+      STDOUT << "Scraped #{parsedPapers.length} from Google Scholar.\n"
+    end
+
+    # Scholar doesn't provide DOI. 
+    # But if registered at crossref (its free), DOI can be retreived. 
+    def getDoi(lastNameFirstAuthor, title, crossRefEmail)
+      return '' if @crossRefEmail.nil?
+      sleep(1) # to reduce risk 
+      STDERR << "Getting DOI for paper by #{lastNameFirstAuthor}: #{title}.\n"
+      url = 'http://www.crossref.org/openurl?redirect=false' +  
+        '&pid='    + crossRefEmail + 
+        '&aulast=' + lastNameFirstAuthor   +
+        '&atitle=' + URI.escape(title)
+      crossRefXML = Nokogiri::XML(open(url)) 
+      crossRefXML.search("doi").children.first.content rescue ''
+    end
+  end
+  
+  class Formatter
+    attr_accessor :parser, :nameToHighlight, :pdfLinks, :altmetricDOIs
+    
+    def initialize(parser, nameToHighlight = nil, pdfLinks = {}, altmetricDOIs = [], minCitationCount = 1)
+      @parser          = parser
+      @nameToHighlight = nameToHighlight
+      @pdfLinks        = pdfLinks
+      @altmetricDOIs   = altmetricDOIs
+      @minCitations    = minCitationCount
+    end
+
+    def to_html
+      ##@doc = Nokogiri::HTML::DocumentFragment.parse "" 
+      builder = Nokogiri::HTML::Builder.new do |doc|
+        doc.html {
+          doc.body {
+            @parser.parsedPapers.each_with_index { |paper, index|
+              doc.div( :class => "publication") {
+                doc.p {
+                  doc.text ((@parser.parsedPapers).length - index).to_s + '. ' 
+
+                  if paper[:authors].include?(@nameToHighlight)
+                    doc.text( paper[:authors].sub(Regexp.new(@nameToHighlight + '.*'), '') )
+                    doc.span( :class => "me") { doc.text @nameToHighlight }
+                    doc.text( paper[:authors].sub(Regexp.new('.*' + @nameToHighlight), '') )
+                  else
+                    doc.text( paper[:authors])
+                  end
+                                    
+                  doc.text ' ' + paper[:year] + '. '
+                  doc.b    paper[:title] + '.'
+                  doc.br
+                  doc.em   paper[:journalName]
+                  doc.text ' '
+                  doc.text paper[:journalDetails]
+                  
+                  unless paper[ :doi].empty?
+                    doc.text(' ')
+                    doc.a( :href => URI.join("http://dx.doi.org/", paper[ :doi]))  { 
+                      doc.text "[DOI]" 
+                    } 
+                  end
+                  if @pdfLinks.keys.include?(paper[:title])
+                    doc.text(' ')
+                    doc.a( :href => @pdfLinks[paper[:title]])  { 
+                      doc.text "[PDF]"
+                    } 
+                  end
+                  if paper[ :citationCount].to_i > @minCitations
+                    doc.text(' ')
+                    doc.a( :href => paper[ :citingPapers]) { 
+                      doc.text("[Cited #{paper[ :citationCount]}x]") 
+                    } 
+                  end
+                  if altmetricDOIs.include?( paper[ :doi])
+                    doc.text(' ')
+                    doc.span( :class                => 'altmetric-embed',
+                              :'data-badge-popover' => 'bottom',
+                              :'data-doi'           => paper[ :doi]        )
+                  end
+                }
+              }
+            }
+          }
+        }
+      end
+      return builder.to_html
+    end
+  end
+end
+
+

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: rubyscholar
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Yannick Wurm
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-08-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+description: A small script to "scrape" your Google Scholar citations and reformat
+  them. It doesn't do a whole lot, but it's still useful.
+email:
+- y.wurm@qmul.ac.uk
+executables:
+- scrape.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- README.md
+- bin/scrape.rb
+- config.yml
+- lib/rubyscholar.rb
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: RubyScholar - Scrape your Google Scholar citations.
+test_files: []