arxivsync 0.0.5 → 0.0.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 51a06a56a4190a2205f27edb93153fe48b4fd361
+  data.tar.gz: c6b1420f327fd406332b0af246fa905bd746aa5d
+SHA512:
+  metadata.gz: e6d87c40b4a3659930dfa316247a95d3a6c70c2ef6d98c839f158e07ff414499b409556507d547e67ef13a70078ca00bbd49b345a457bfaff0ff7b8c6d98594f
+  data.tar.gz: c5aaa89769f8b0efe19cdcfd18ac8e7a92d5e162f299614372840573f2505fb09e8872a7527434ca05a08db91e072414583b3794cacebadafa3cce0fc1c6c830

data/Gemfile

@@ -1,3 +1,4 @@
+ruby '2.1.0'
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in arxivsync.gemspec

data/README.md

@@ -1,4 +1,4 @@
-# arxivsync-0.0.5
+# arxivsync 0.0.6
 
 Ruby OAI interface for harvesting the arXiv. Can be used to store and update an XML mirror of paper metadata, and parse the XML into Ruby objects to allow conversion into a friendlier format.
 
@@ -12,31 +12,35 @@ Ruby OAI interface for harvesting the arXiv. Can be used to store and update an
 
 ### Creating or updating an archive
 
-```
-  archive = ArxivSync::XMLArchive.new("/home/foo/savedir")
-  archive.sync
+Use the included shell command:
+
+```bash
+  arxivsync ARCHIVE_DIR
 ```
 
-Stores each XML response as an individual file, each containing up to 1000 records. Following an initial harvest, you can rerun this to add additional files containing all records since the last harvest.
+This stores each XML response as an individual file, each containing up to 1000 records. Following an initial harvest, you can rerun this to add additional files containing all records since the last harvest.
 
 Remember to leave at least a day between syncs-- the temporal granularity doesn't go any smaller than that!
 
 ### Reading from an archive
 
-```
+```ruby
   archive = ArxivSync::XMLArchive.new("/home/foo/savedir")
-  archive.read_metadata do |paper|
-    # Do stuff with paper
+  archive.read_metadata do |papers|
+    # Papers come in blocks of at most 1000 at a time
+    papers.each do |paper|
+      # Do stuff with papers
+    end
   end
 ```
 
-Parses the XML files using Nokogiri's SAX parser and yields Structs representing the metadata as it goes.
+Parses the XML files using a SAX parser and yields Structs representing the metadata as it goes. The structures returned will closely match the [arxivRaw](http://export.arxiv.org/oai2?verb=GetRecord&identifier=oai:arXiv.org:0804.2273&metadataPrefix=arXivRaw) format.
 
 ### Download and parse immediately
 
 If you just want arxivsync to do the request-cycle and parsing bits but handle storage yourself:
 
-```
+```ruby
   ArxivSync.get_metadata(oai_params) do |resp, papers|
     papers.each do |paper|
       # Do stuff with paper

data/Rakefile

@@ -1 +1,6 @@
+require 'rake/testtask'
 require "bundler/gem_tasks"
+
+Rake::TestTask.new do |t|
+  t.pattern = "test/**/*_test.rb"
+end

data/arxivsync.gemspec

@@ -21,8 +21,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
+  spec.add_development_dependency "minitest-reporters"
 
   spec.add_runtime_dependency "oai"
-  spec.add_runtime_dependency "nokogiri"
-  spec.add_runtime_dependency "ox", ">= 2.0.2"
+  spec.add_runtime_dependency "colorize"
+  spec.add_runtime_dependency "ox", ">= 2.0.2" # Super-fast XML parser
+  spec.add_runtime_dependency "nokogiri" # Slower but more accurate parser
 end

data/bin/arxivsync

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require 'arxivsync'
+require 'optparse'
+
+def arxivsync
+  optparse = OptionParser.new do |opts|
+    opts.banner = %Q{Usage: arxivsync ARCHIVE_DIR
+
+Begin or resume syncing a full copy of the arXiv
+metadata to ARCHIVE_DIR. The directory will be created 
+if it does not exist.}
+  end
+
+  optparse.parse!
+
+  if ARGV.empty?
+    puts optparse
+    exit 1
+  end
+
+  archive = ArxivSync::XMLArchive.new(ARGV[0])
+  archive.sync
+end
+
+if __FILE__ == $0
+  arxivsync
+end

data/lib/arxivsync/downloader.rb

@@ -4,12 +4,12 @@ module ArxivSync
       @initial_params = initial_params
 
       if @initial_params[:from] == Date.today
-        puts "Last responseDate was today. arXiv lacks date granularity beyond the day level; please wait before continuing harvest."
+        puts "Last responseDate was today. arXiv lacks date granularity beyond the day level; please wait before continuing harvest.".light_yellow
         return false
       end
 
       unless @initial_params[:resumptionToken]
-        @initial_params[:metadataPrefix] ||= 'arXiv'
+        @initial_params[:metadataPrefix] ||= 'arXivRaw'
       end
       @last_params = nil
 
@@ -25,12 +25,12 @@ module ArxivSync
       while true
         if !resp.resumption_token || resp.resumption_token.empty?
           if resp.doc.to_s.include?("Retry after 20 seconds") # Rate limitation
-            puts "Honoring 503 and sleeping for 20 seconds..."
+            puts "Honoring 503 and sleeping for 20 seconds...".light_yellow
             sleep 20
             resp = retry_request
           else # No resumption_token and no retry should mean we're finished
             b.call(resp)
-            puts "Finished archiving!"
+            puts "Finished archiving~!".bold.light_green
             break
           end
         else # We have a resumption_token, keep going!
@@ -47,14 +47,14 @@ module ArxivSync
     end
 
     def make_request(params)
-      puts "Making OAI request with params: #{params.inspect}"
+      puts "Making OAI request with params: #{params.inspect}".light_magenta
 
       @last_params = params.clone # list_records will nuke our params
 
       begin
         return @oai.list_records(params)
       rescue Faraday::Error::TimeoutError
-        puts "Request timed out; retrying in 20 seconds"
+        puts "Request timed out; retrying in 20 seconds".light_yellow
         sleep 20
         return retry_request
       end

data/lib/arxivsync/parser.rb

@@ -1,15 +1,27 @@
 module ArxivSync
   # Layout reference: http://www.xmlns.me/?op=visualize&id=643
-  Author = Struct.new(
-    :keyname, :forenames, :suffix, :affiliation
+  Version = Struct.new(
+    :date, # Time.parse("Wed, 23 Jan 2008 21:06:41 GMT")
+    :size # '121kb'
   )
 
   Paper = Struct.new(
-    :id, :created, :updated, :authors, :title,
-    :msc_class, :report_no, :journal_ref, :comments,
-    :abstract, :categories, :doi, :proxy, :license,
+    :id, # '0801.3673'
+    :submitter, # 'N. C. Bacalis'
+    :versions,
+    :title, # "Variational Functionals for Excited States"
+    :authors, # ['Naoum C. Bacalis']
+    :categories, # ['quant-ph'] (primary category first, then crosslists)
+    :abstract, # "Functionals that have local minima at the excited..."
 
-    :primary_category, :crosslists
+    # These properties do not always appear
+    :comments, # "4 pages"
+    :msc_class, # '57M25'
+    :report_no, # "LA-UR 07-7165"
+    :journal_ref, # "JHEP 0805:087,2008."
+    :doi, # "10.1088/1126-6708/2008/05/087"
+    :proxy, # "ccsd hal-00214270"
+    :license # "http://arxiv.org/licenses/nonexclusive-distrib/1.0/"
   )
 
   class XMLParser < ::Ox::Sax
@@ -22,9 +34,9 @@ module ArxivSync
         @papers = []
       when :metadata
         @model = Paper.new
-        @authors = []
-      when :author
-        @author = Author.new
+        @versions = []
+      when :version
+        @version = Version.new
       end
     end
 
@@ -34,53 +46,51 @@ module ArxivSync
 
     def text(str)
       case @el
+      # Necessary elements
       when :id
-        @model.id = str
-      when :created
-        @model.created = Date.parse(str)
-      when :updated
-        @model.updated = Date.parse(str)
+        @model.id = clean(str)
+      when :submitter
+        @model.submitter = clean(str)
       when :title
         @model.title = clean(str)
+      when :authors
+        @model.authors = clean(str).split(/,| and /)
+          .map { |s| clean(s) }.reject { |s| s.empty? }
+      when :categories
+        @model.categories = clean(str).split(/\s/)
+      when :abstract
+        @model.abstract = clean(str)
+
+      # Optional elements
+      when :comments
+        @model.comments = clean(str)
       when :"msc-class"
-        @model.msc_class = str
+        @model.msc_class = clean(str)
       when :"report-no"
-        @model.report_no = str
+        @model.report_no = clean(str)
       when :"journal-ref"
-        @model.journal_ref = str
-      when :comments
-        @model.comments = clean(str)
-      when :abstract
-        @model.abstract = clean(str)
-      when :categories
-        @model.categories = str.split
-        @model.primary_category = str.split[0]
-        @model.crosslists = str.split.drop(1)
+        @model.journal_ref = clean(str)
       when :doi
-        @model.doi = str
+        @model.doi = clean(str)
       when :proxy
-        @model.proxy = str
+        @model.proxy = clean(str)
       when :license
-        @model.license = str
-      when :keyname
-        @author.keyname = str
-      when :forenames
-        @author.forenames = str
-      when :suffix
-        @author.suffix = str
-      when :affiliation
-        @author.affiliation = str
+        @model.license = clean(str)
+
+      # Versions
+      when :date
+        @version.date = Time.parse(clean(str))
+      when :size
+        @version.size = clean(str)
       end
     end
 
     def end_element(name)
       case name
-      when :author
-        @authors.push(@author)
+      when :version
+        @versions.push(@version)
       when :metadata # End of a paper entry
-        #@paper.updated_date ||= @paper.pubdate # If no separate updated date
-        #@paper.feed_id = Feed.get_or_create(@primary_category).id
-        @model.authors = @authors
+        @model.versions = @versions
 
         @papers.push(@model)
       end

data/lib/arxivsync/version.rb

@@ -1,3 +1,3 @@
 module ArxivSync
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/lib/arxivsync/xmlarchive.rb

@@ -1,16 +1,25 @@
 module ArxivSync
   class XMLArchive
     def initialize(savedir, custom_params=nil)
-      @savedir = savedir
-      begin Dir.mkdir(@savedir) # Ensure this directory exists
-      rescue Errno::EEXIST
+      @savedir = File.expand_path(savedir)
+
+      if not Dir.exists?(@savedir)
+        puts "Creating new XML archive at #{@savedir}\n".light_green
+        Dir.mkdir(@savedir)
       end
     end
 
     # Parse the timestamp from the path to a previously saved
     # arxiv xml block
     def parse_dt(path)
-      DateTime.parse(path.split('/')[-1].split('_')[0])
+      begin
+        DateTime.parse(path.split('/')[-1].split('_')[0])
+      rescue ArgumentError
+        puts "Failed to parse timestamp from file #{path}\n".bold.light_red
+        puts ("Are you sure this is an archive directory?\n" +
+             "If so, it needs to be free of strange interloping files.").bold.light_white
+        exit 1
+      end
     end
 
     # Download from the arXiv!
@@ -31,9 +40,10 @@ module ArxivSync
       end
 
       if existing.empty?
-        puts "Commencing full arXiv download. This will take quite a while.\n" +
+        puts ("Commencing full arXiv download. This will take ... a while.\n" +
              "Download can be safely aborted at any point and will resume from\n" +
-             "last successful response."
+             "last successful response. However, resumptionTokens *will* expire\n" +
+             "if you leave it in an incomplete state for long enough.\n").bold.light_white
       else
         # Parse the most recent one
         last_response = Nokogiri::XML(File.open(existing[-1]))
@@ -41,10 +51,10 @@ module ArxivSync
 
         if last_token.empty? # Previous sync completed successfully
           responseDate = Date.parse(last_response.css('responseDate').text)
-          puts "Downloading from last responseDate: #{responseDate}"
+          puts "Downloading from last responseDate: #{responseDate}\n".bold.light_green
           oai_params[:from] = responseDate
         else # Previous sync aborted prematurely, resume
-          puts "Resuming download using previous resumptionToken: #{last_token}"
+          puts "Resuming download using previous resumptionToken: #{last_token}\n".bold.light_green
           oai_params = { resumptionToken: last_token }
         end
       end
@@ -97,7 +107,7 @@ module ArxivSync
       f = File.open("#{@savedir}/#{filename}", 'w')
       f.write(content)
       f.close
-      puts "Saved #{cursor+numRecords}/#{completeListSize} records to #{filename}"
+      puts "Saved #{cursor+numRecords} of #{completeListSize} records to #{filename}".light_green
     end
   end
 end

data/lib/arxivsync.rb

@@ -1,6 +1,7 @@
 require 'oai'
 require 'nokogiri'
 require 'ox'
+require 'colorize'
 require 'arxivsync/version'
 require 'arxivsync/parser'
 require 'arxivsync/downloader'