arxivsync 0.0.9 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9125d2dc0e6a12de5a4ec015f45d12801189090f
-  data.tar.gz: 71ad86f611dc9153ee668580078840109e2d8394
+  metadata.gz: dddf2ff2529dee6f0d6638d7f58096c9a775b9b0
+  data.tar.gz: 624f0ef9a81a0c3cb3990736181eaef64c845caa
 SHA512:
-  metadata.gz: 8fc09072898e94ce1903df950d87fc4ab2a3ef737355f014afc473ca9e28aa6f154c74e7622be6f0097fbd2284d76645c7bbbcbf2b7a0c61ecd7faa75463cbf2
-  data.tar.gz: f19f459fa163cac5044457fc9aa88355cd6d9faecc04bbe4cb67f28b411fbef0ef55c892c47a26c8b97811fa8b9237e8194f3ea4b63f1be39c20dcfce57145c0
+  metadata.gz: 107a2b4336920bf2a134e7a4ca04091b596199f9c984f84b3a3440062dc1f02d37e5e0d244735550ebea7000a6bfd6b607fe0e423544b892a35d964153b47539
+  data.tar.gz: e27c824d500aa07108813cd14e1fadc6b78c308f2c2302ec1e9983392efd89ac4137d8021ccb73492b447f2bb4bfc34f3e0fa370ef20c71d806ab57e7d0c3c8b

data/README.md

@@ -1,4 +1,4 @@
-# arxivsync 0.0.9
+# arxivsync 0.1.0
 
 Ruby OAI interface for harvesting the arXiv. Can be used to store and update an XML mirror of paper metadata, and parse the XML into Ruby objects to allow conversion into a friendlier format.
 

data/lib/arxivsync/parser.rb

@@ -10,6 +10,7 @@ module ArxivSync
     :submitter, # 'N. C. Bacalis'
     :versions,
     :title, # "Variational Functionals for Excited States"
+    :author_str, # "Naoum C. Bacalis"
     :authors, # ['Naoum C. Bacalis']
     :categories, # ['quant-ph'] (primary category first, then crosslists)
     :abstract, # "Functionals that have local minima at the excited..."
@@ -48,13 +49,48 @@ module ArxivSync
       str.gsub(/\s+/, ' ').strip
     end
 
+    # Like LaTeX.decode but without the punctuation weirdness
+    def latex_decode(str)
+      string = str.dup
+
+      LaTeX::Decode::Base.normalize(string)
+
+      LaTeX::Decode::Maths.decode!(string)
+
+      LaTeX::Decode::Accents.decode!(string)
+      LaTeX::Decode::Diacritics.decode!(string)
+      #LaTeX::Decode::Punctuation.decode!(string)
+      LaTeX::Decode::Symbols.decode!(string)
+
+      LaTeX::Decode::Base.strip_braces(string)
+
+      LaTeX.normalize_C(string)
+    end
+
     def decode(string)
       str = @entities.decode(string)
-      LaTeX::Decode::Base.normalize(str)
-      LaTeX::Decode::Accents.decode!(str)
-      LaTeX::Decode::Diacritics.decode!(str)
-      LaTeX::Decode::Symbols.decode!(str)
-      str
+
+      # Process latex entities -- except inside equations
+      decoded = ""
+      equation = false
+      segment = ""
+      str.chars do |ch|
+        if ch == '$' 
+          if !equation
+            decoded << latex_decode(segment)
+            segment = ch
+          else
+            decoded << segment + ch
+            segment = ""
+          end
+
+          equation = !equation
+        else
+          segment << ch
+        end
+      end
+
+      decoded << latex_decode(segment)
     end
 
     def text(str)
@@ -69,10 +105,12 @@ module ArxivSync
       when :authors
         # Author strings may contain strange metadata
         # Non-regex parsing to handle nested parens
+        @model.author_str = decode(clean(str))
+
         depth = 0
         no_parens = ""
 
-        str.chars do |ch|
+        @model.author_str.chars do |ch|
           case ch
           when '('
             depth += 1
@@ -83,8 +121,8 @@ module ArxivSync
           end
         end
 
-        @model.authors = clean(no_parens).split(/,| and /)
-          .map { |s| decode(clean(s)) }
+        @model.authors = no_parens.split(/,|:|;|\sand\s|\s?the\s/i)
+          .map { |s| clean(s) }
           .reject { |s| s.empty? }
       when :categories
         @model.categories = clean(str).split(/\s/)

data/lib/arxivsync/version.rb

@@ -1,3 +1,3 @@
 module ArxivSync
-  VERSION = "0.0.9"
+  VERSION = "0.1.0"
 end

data/test/parser_test.rb

@@ -13,19 +13,20 @@ class TestParser < Minitest::Test
       assert_equal papers.count, 1000
       papers.each do |paper|
         if paper.id == '0801.3673'
-          assert_equal paper.submitter, "N. C. Bacalis"
+          assert_equal "N. C. Bacalis", paper.submitter
 
-          assert_equal paper.versions.length, 1
-          assert_equal paper.versions[0].date, Time.parse("Wed, 23 Jan 2008 21:06:41 GMT")
-          assert_equal paper.versions[0].size, "121kb"
+          assert_equal 1, paper.versions.length
+          assert_equal Time.parse("Wed, 23 Jan 2008 21:06:41 GMT"), paper.versions[0].date
+          assert_equal "121kb", paper.versions[0].size
 
-          assert_equal paper.title, "Variational Functionals for Excited States"
+          assert_equal "Variational Functionals for Excited States", paper.title
+          assert_equal "Naoum C. Bacalis", paper.author_str
 
-          assert_equal paper.authors, ["Naoum C. Bacalis"]
-          assert_equal paper.categories, ["quant-ph"]
+          assert_equal ["Naoum C. Bacalis"], paper.authors
+          assert_equal ["quant-ph"], paper.categories
 
-          assert_equal paper.comments, "4 pages"
-          assert_equal paper.abstract, "Functionals that have local minima at the excited states of a non degenerate Hamiltonian are presented. Then, improved mutually orthogonal approximants of the ground and the first excited state are reported."
+          assert_equal "4 pages", paper.comments
+          assert_equal "Functionals that have local minima at the excited states of a non degenerate Hamiltonian are presented. Then, improved mutually orthogonal approximants of the ground and the first excited state are reported.", paper.abstract
           tested += 1
         end
 
@@ -39,8 +40,8 @@ class TestParser < Minitest::Test
           assert_equal paper.versions[1].size, "58kb"
           
           assert_equal paper.title, "Weak Localization of Dirac Fermions in Graphene"
-          assert_equal paper.authors, ["Xin-Zhong Yan", "C. S. Ting"]
-          assert_equal paper.categories, ["cond-mat.str-el"]
+          assert_equal ["Xin-Zhong Yan", "C. S. Ting"], paper.authors
+          assert_equal ["cond-mat.str-el"], paper.categories
           assert_equal paper.comments, "4 pages, 4 figures"
           assert_equal paper.journal_ref, "PRL 101, 126801 (2008)"
           assert_equal paper.doi, "10.1103/PhysRevLett.101.126801"
@@ -50,11 +51,11 @@ class TestParser < Minitest::Test
 
         # Ensure we handle TeX special characters
         if paper.id == "0801.3763"
-          assert_equal "Dijana Žilić", paper.authors[1]
+          assert_equal "Dijana Žilić", paper.authors[1]
 
           # But make sure we didn't try to parse any
           # complex math-- that cannot be unicode
-          assert_includes paper.abstract, "[Cu(bpy)$_3$]$_2$[Cr(C$_2$O$_4$)$_3$]NO$_3⋅$9H$_2$O"
+          assert_includes paper.abstract, "[Cu(bpy)$_3$]$_2$[Cr(C$_2$O$_4$)$_3$]NO$_3\\cdot $9H$_2$O"
         end
 
         # Ensure we parse html entities

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arxivsync
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.1.0
 platform: ruby
 authors:
 - Jaiden Mispy