arxivsync 0.0.3

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in arxivsync.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Jaiden Mispy
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,53 @@
+# arxivsync-0.0.3
+
+Ruby OAI interface for harvesting the arXiv. Can be used to store and update an XML mirror of paper metadata, and parse the XML into Ruby objects to allow conversion into a friendlier format.
+
+## Installation
+
+```
+  gem install arxivsync
+```
+
+## Usage
+
+### Creating or updating an archive
+
+```
+  archive = ArxivSync::XMLArchive.new("/home/foo/savedir")
+  archive.sync
+```
+
+Stores each XML response as an individual file, each containing up to 1000 records. Following an initial harvest, you can rerun this to add additional files containing all records since the last harvest.
+
+Remember to leave at least a day between syncs-- the temporal granularity doesn't go any smaller than that!
+
+### Reading from an archive
+
+```
+  archive = ArxivSync::XMLArchive.new("/home/foo/savedir")
+  archive.read_metadata do |paper|
+    # Do stuff with paper
+  end
+```
+
+Parses the XML files using Nokogiri's SAX parser and yields Structs representing the metadata as it goes.
+
+### Download and parse immediately
+
+If you just want arxivsync to do the request-cycle and parsing bits but handle storage yourself:
+
+```
+  ArxivSync.get_metadata(oai_params) do |resp, papers|
+    papers.each do |paper|
+      # Do stuff with paper
+    end 
+  end
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/arxivsync.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'arxivsync/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "arxivsync"
+  spec.version       = ArxivSync::VERSION
+  spec.authors       = ["Jaiden Mispy"]
+  spec.email         = ["scirate@mispy.me"]
+  spec.description   = %q{OAI interface for harvesting the arXiv database}
+  spec.summary       = %q{OAI interface for harvesting the arXiv database}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
+
+  spec.add_runtime_dependency "oai"
+  spec.add_runtime_dependency "nokogiri"
+  spec.add_runtime_dependency "ox", ">= 2.0.2"
+end

data/lib/arxivsync/downloader.rb

@@ -0,0 +1,57 @@
+module ArxivSync
+  class Downloader
+    def initialize(initial_params={})
+      @initial_params = initial_params
+      unless @initial_params[:resumptionToken]
+        @initial_params[:metadataPrefix] ||= 'arXiv'
+      end
+      @last_params = nil
+
+      @oai = OAI::Client.new('http://export.arxiv.org/oai2', :parser => 'libxml')
+    end
+
+    def start(&b)
+      # Make the initial request
+      resp = make_request(@initial_params)
+
+      # Continue to make requests until the server stops sending
+      # resumption tokens
+      while true
+        if !resp.resumption_token || resp.resumption_token.empty?
+          if resp.doc.to_s.include?("Retry after 20 seconds") # Rate limitation
+            puts "Honoring 503 and sleeping for 20 seconds..."
+            sleep 20
+            resp = retry_request
+          else # No resumption_token and no retry should mean we're finished
+            b.call(resp)
+            puts "Finished archiving!"
+            break
+          end
+        else # We have a resumption_token, keep going!
+          b.call(resp)
+          resp = make_request(resumptionToken: resp.resumption_token)
+        end
+      end
+
+      return self
+    end
+
+    def retry_request
+      make_request(@last_params)
+    end
+
+    def make_request(params)
+      puts "Making OAI request with params: #{params.inspect}"
+
+      @last_params = params.clone # list_records will nuke our params
+
+      begin
+        return @oai.list_records(params)
+      rescue Faraday::Error::TimeoutError
+        puts "Request timed out; retrying in 20 seconds"
+        sleep 20
+        return retry_request
+      end
+    end
+  end
+end

data/lib/arxivsync/parser.rb

@@ -0,0 +1,67 @@
+module ArxivSync
+  Author = Struct.new(
+    :keyname, :forenames
+  )
+
+  Paper = Struct.new(
+    :id, :created, :updated, :title, :abstract, :authors,
+    :primary_category, :crosslists
+  )
+
+  class XMLParser < ::Ox::Sax
+    attr_accessor :papers
+
+    def start_element(name, attributes=[])
+      @el = name
+      case name
+      when :ListRecords
+        @papers = []
+      when :metadata
+        @model = Paper.new
+        @authors = []
+      when :author
+        @author = Author.new
+      end
+    end
+
+    def clean(str)
+      str.gsub(/\s+/, ' ').strip
+    end
+
+    def text(str)
+      case @el
+      when :id
+        @model.id = str
+      when :created
+        @model.created = Date.parse(str)
+      when :updated
+        @model.updated = Date.parse(str)
+      when :title
+        @model.title = clean(str)
+      when :abstract
+        @model.abstract = clean(str)
+      when :categories
+        @model.primary_category = str.split[0]
+        @model.crosslists = str.split.drop(1)
+      when :keyname
+        @author.keyname = str
+      when :forenames
+        @author.forenames = str
+      end
+    end
+
+    def end_element(name)
+      case name
+      when :author
+        @authors.push(@author)
+      when :metadata # End of a paper entry
+        #@paper.updated_date ||= @paper.pubdate # If no separate updated date
+        #@paper.feed_id = Feed.get_or_create(@primary_category).id
+        @model.authors = @authors
+
+        @papers.push(@model)
+      end
+      @el = nil
+    end
+  end
+end

data/lib/arxivsync/version.rb

@@ -0,0 +1,3 @@
+module ArxivSync
+  VERSION = "0.0.3"
+end

data/lib/arxivsync/xmlarchive.rb

@@ -0,0 +1,107 @@
+module ArxivSync
+  class XMLArchive
+    def initialize(savedir, custom_params=nil)
+      @savedir = savedir
+      begin Dir.mkdir(@savedir) # Ensure this directory exists
+      rescue Errno::EEXIST
+      end
+    end
+
+    # Parse the timestamp from the path to a previously saved
+    # arxiv xml block
+    def parse_dt(path)
+      DateTime.parse(path.split('/')[-1].split('_')[0])
+    end
+
+    # Download from the arXiv!
+    # This can be called in three potential states:
+    # - The savedir has yet to be populated with any xml, meaning we need to
+    #   start a full mirror of the entire database.
+    #
+    # - The most recent xml file contains a resumptionToken, meaning the last
+    #   harvest attempt was aborted prematurely and we need to resume.
+    #
+    # - The most recent xml file does not have a resumptionToken, in which case
+    #   we begin a new harvest for everything since the responseDate of the last.
+    def sync(oai_params={})
+      # Find any existing xml files we may have, sorted by
+      # responseDate in the filename
+      existing = Dir.glob(File.join(@savedir, '*')).sort do |a,b|
+        parse_dt(a) <=> parse_dt(b)
+      end
+
+      if existing.empty?
+        puts "Commencing full arXiv download. This will take quite a while.\n" +
+             "Download can be safely aborted at any point and will resume from\n" +
+             "last successful response."
+      else
+        # Parse the most recent one
+        last_response = Nokogiri::XML(File.open(existing[-1]))
+        last_token = last_response.css('resumptionToken').text
+
+        if last_token.empty? # Previous sync completed successfully
+          responseDate = Date.parse(last_response.css('responseDate').text)
+          if responseDate == Date.today
+            puts "Last responseDate was today. arXiv lacks date granularity beyond the day level; please wait before continuing harvest."
+            return false
+          end
+          puts "Downloading from last responseDate: #{responseDate}"
+          oai_params[:from] ||= responseDate
+        else # Previous sync aborted prematurely, resume
+          puts "Resuming download using previous resumptionToken: #{last_token}"
+          oai_params = { resumptionToken: last_token }
+        end
+      end
+
+      downloader = Downloader.new(oai_params)
+      downloader.start do |resp|
+        save_response(resp)
+      end
+    end
+
+    # Parses the archive using Nokogiri's SAX parser
+    # Yields Paper objects as they are created
+    def read_metadata(&b)
+      parser = XMLParser.new
+
+      Dir.glob(File.join(@savedir, '*')).each do |path|
+        Ox.sax_parse(parser, File.open(path))
+        b.call(parser.papers)
+      end
+    end
+
+    # Saves a timestamped OAI XML response to disk, appending
+    # the resumption token to the filename if available
+    def save_response(resp)
+      content = resp.doc.to_s
+
+      # Parse the response and extract some metadata
+      doc = Nokogiri::XML(content)
+
+      # responseDate for stamping files and potentially
+      # initiating the next harvest
+      responseDate = doc.css('responseDate').text
+
+      # Total number of records in this harvest
+      completeListSize = doc.css('resumptionToken').attr('completeListSize').value.to_i
+      # How far we are in
+      cursor = doc.css('resumptionToken').attr('cursor').value.to_i
+      # How many records we gained in this response
+      numRecords = doc.css('record').count.to_i
+
+      # If we have a resumption_token, stick that on the filename.
+      if resp.resumption_token && !resp.resumption_token.empty?
+        suffix = resp.resumption_token
+      else
+        suffix = 'final'
+      end
+
+      # Write out the file and communicate progress
+      filename = "#{responseDate}_#{suffix}"
+      f = File.open("#{@savedir}/#{filename}", 'w')
+      f.write(content)
+      f.close
+      puts "Saved #{cursor+numRecords}/#{completeListSize} records to #{filename}"
+    end
+  end
+end

data/lib/arxivsync.rb

@@ -0,0 +1,24 @@
+require 'oai'
+require 'nokogiri'
+require 'ox'
+require 'arxivsync/version'
+require 'arxivsync/parser'
+require 'arxivsync/downloader'
+require 'arxivsync/xmlarchive'
+
+module ArxivSync
+  class << self
+    def parse_xml(xml)
+      parser = XMLParser.new
+      Ox.sax_parse(parser, StringIO.new(xml))
+      parser.papers
+    end
+
+    def get_metadata(oai_params, &b)
+      downloader = Downloader.new(oai_params)
+      downloader.start do |resp|
+        b.call(resp, parse_xml(resp.doc.to_s))
+      end
+    end
+  end
+end