arxivsync 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6f23b2225a55b6369e6c84961b6328247289573b
-  data.tar.gz: dfc6064821a4a6d7c12434cff5ae551c4f784600
+  metadata.gz: 9125d2dc0e6a12de5a4ec015f45d12801189090f
+  data.tar.gz: 71ad86f611dc9153ee668580078840109e2d8394
 SHA512:
-  metadata.gz: 44ccf8d910693b46719262f0778b76e9933a5d7bd521bd75dfb6c1a17205f8efc61d2549f0e7a16d62b2b4b74ff966df5b52ece62997219e4251595ad90f0054
-  data.tar.gz: 249f809cfc44b4bf32a5fadf072bc87be91a98fb6a4fcf23f1dddd00f171402cf78fdb44e44fd23b3bcf11d49bcedbc8cd28ba872d4e89f3f1760845ec0feb68
+  metadata.gz: 8fc09072898e94ce1903df950d87fc4ab2a3ef737355f014afc473ca9e28aa6f154c74e7622be6f0097fbd2284d76645c7bbbcbf2b7a0c61ecd7faa75463cbf2
+  data.tar.gz: f19f459fa163cac5044457fc9aa88355cd6d9faecc04bbe4cb67f28b411fbef0ef55c892c47a26c8b97811fa8b9237e8194f3ea4b63f1be39c20dcfce57145c0

data/README.md

@@ -1,4 +1,4 @@
-# arxivsync 0.0.8
+# arxivsync 0.0.9
 
 Ruby OAI interface for harvesting the arXiv. Can be used to store and update an XML mirror of paper metadata, and parse the XML into Ruby objects to allow conversion into a friendlier format.
 

data/arxivsync.gemspec

@@ -25,7 +25,8 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "oai"
   spec.add_runtime_dependency "colorize"
-  spec.add_runtime_dependency "latex-decode"
+  spec.add_runtime_dependency "htmlentities", "~> 4.3.1"
+  spec.add_runtime_dependency "latex-decode", "~> 0.1.1"
   spec.add_runtime_dependency "ox", ">= 2.0.2" # Super-fast XML parser
   spec.add_runtime_dependency "nokogiri" # Slower but more accurate parser
 end

data/lib/arxivsync/parser.rb

@@ -27,6 +27,10 @@ module ArxivSync
   class XMLParser < ::Ox::Sax
     attr_accessor :papers
 
+    def initialize
+      @entities = HTMLEntities.new
+    end
+
     def start_element(name, attributes=[])
       @el = name
       case name
@@ -34,7 +38,7 @@ module ArxivSync
         @papers = []
       when :metadata
         @model = Paper.new
-        @versions = []
+        @model.versions = []
       when :version
         @version = Version.new
       end
@@ -44,27 +48,52 @@ module ArxivSync
       str.gsub(/\s+/, ' ').strip
     end
 
+    def decode(string)
+      str = @entities.decode(string)
+      LaTeX::Decode::Base.normalize(str)
+      LaTeX::Decode::Accents.decode!(str)
+      LaTeX::Decode::Diacritics.decode!(str)
+      LaTeX::Decode::Symbols.decode!(str)
+      str
+    end
+
     def text(str)
       case @el
       # Necessary elements
       when :id
         @model.id = clean(str)
       when :submitter
-        @model.submitter = clean(str)
+        @model.submitter = decode(clean(str))
       when :title
-        @model.title = clean(str)
+        @model.title = decode(clean(str))
       when :authors
-        regex = /(?:,| and )(?![^\(\)]*+\))/
-        @model.authors = clean(str).split(regex)
-          .map { |s| LaTeX.decode(clean(s)) }.reject { |s| s.empty? }
+        # Author strings may contain strange metadata
+        # Non-regex parsing to handle nested parens
+        depth = 0
+        no_parens = ""
+
+        str.chars do |ch|
+          case ch
+          when '('
+            depth += 1
+          when ')'
+            depth -= 1
+          else
+            no_parens << ch if depth == 0
+          end
+        end
+
+        @model.authors = clean(no_parens).split(/,| and /)
+          .map { |s| decode(clean(s)) }
+          .reject { |s| s.empty? }
       when :categories
         @model.categories = clean(str).split(/\s/)
       when :abstract
-        @model.abstract = clean(str)
+        @model.abstract = decode(clean(str))
 
       # Optional elements
       when :comments
-        @model.comments = clean(str)
+        @model.comments = decode(clean(str))
       when :"msc-class"
         @model.msc_class = clean(str)
       when :"report-no"
@@ -89,10 +118,8 @@ module ArxivSync
     def end_element(name)
       case name
       when :version
-        @versions.push(@version)
+        @model.versions.push(@version)
       when :metadata # End of a paper entry
-        @model.versions = @versions
-
         @papers.push(@model)
       end
       @el = nil

data/lib/arxivsync/version.rb

@@ -1,3 +1,3 @@
 module ArxivSync
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/lib/arxivsync.rb

@@ -2,7 +2,9 @@ require 'oai'
 require 'nokogiri'
 require 'ox'
 require 'colorize'
+require 'htmlentities'
 require 'latex/decode'
+
 require 'arxivsync/version'
 require 'arxivsync/parser'
 require 'arxivsync/downloader'

data/test/parser_test.rb

@@ -48,9 +48,30 @@ class TestParser < Minitest::Test
           tested += 1
         end
 
-        if paper.id == "0801.3713"
-          # Ensure latex-decode works properly
-          assert_equal "Jean-Claude Léon (LGS)", paper.authors[2]
+        # Ensure we handle TeX special characters
+        if paper.id == "0801.3763"
+          assert_equal "Dijana Žilić", paper.authors[1]
+
+          # But make sure we didn't try to parse any
+          # complex math-- that cannot be unicode
+          assert_includes paper.abstract, "[Cu(bpy)$_3$]$_2$[Cr(C$_2$O$_4$)$_3$]NO$_3⋅$9H$_2$O"
+        end
+
+        # Ensure we parse html entities
+        if paper.id == "0801.3778"
+          assert_equal "6 pages, 10 figures, to appear in \"Young massive clusters, initial conditions and environments\", typo in author's name corrected", paper.comments
+        elsif paper.id == "0801.3789"
+          assert_includes paper.abstract, "The addition of this \"conservative noise\" allows"
+        end
+
+        # And weird author strings
+        if paper.id == "0801.3898"
+          assert_equal ["A. Frasca", "Zs. Kovari", "K.G. Strassmeier", "K. Biazzo"], paper.authors
+        end
+
+        # And those pesky "and"s
+        if paper.id == "0801.3674"
+          assert_equal ["Robert H. Brandenberger", "Keshav Dasgupta", "Anne-Christine Davis"], paper.authors
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arxivsync
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Jaiden Mispy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-27 00:00:00.000000000 Z
+date: 2014-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,20 +94,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.3.1
 - !ruby/object:Gem::Dependency
   name: latex-decode
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: ox
   requirement: !ruby/object:Gem::Requirement