anystyle 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d07c8a520550b0faa3e7a9b3c5b7368fdce46c02
-  data.tar.gz: 4256158f3ed52621a10e25147bab967c8097578d
+SHA256:
+  metadata.gz: 4a8c6471369e8969b190536c6f597742e6917197ce26009b1f7c7dcd1ec32168
+  data.tar.gz: 1f9af1e1337c47fda651b40fd19903d7474fe860b679bcafd832b932fc075947
 SHA512:
-  metadata.gz: 22c6e55e7b9bbb2cbfa36ef0b1ffc72fe117b2298e4a82b74369a0d9834476fb925222b1a0c6df5ed97cc802db7878a52c837295dc7932a90041544dc944ef5d
-  data.tar.gz: 07edabc567099d9ecd3bcaf3b0461912eb740f340525dd57d5062edb398db6b285f95f12da83ed762514cb4ebf58cbb8ac13a643d631b8a346e73b46fd182a4a
+  metadata.gz: 504a133a3cefedeb24fc9c165e00c9f59f6dfee14b78fa0206b9eadb2d060e236b8e99fe211d94d8df1de9d9c93efc8d3149c931827c5df83bfa56524539ce55
+  data.tar.gz: 70a9e1c156fe996980610755971eb4feb899afd2a57e20cbd95c664618326385b88ef8c733268922977a7252b2fff47771ba69892f37eecc94227545c4e956b3

data/HISTORY.md

@@ -1,3 +1,11 @@
+1.3.0 / 2018-09-18
+==================
+* Updated and improved normalizers and CSL format.
+* Improved Chinese reference tokenization.
+* Added option to customizee pdftotext path.
+* Improved Finder reference line joining.
+* Improved Finder model; training sets.
+* Improved Parser model; training sets.
 1.2.1 / 2018-08-17
 ==================
 * Added check and train commands to CLI.

data/lib/anystyle.rb

@@ -28,6 +28,7 @@ require 'anystyle/feature/terminal'
 require 'anystyle/feature/words'
 
 require 'anystyle/normalizer'
+require 'anystyle/normalizer/arxiv'
 require 'anystyle/normalizer/brackets'
 require 'anystyle/normalizer/container'
 require 'anystyle/normalizer/date'

data/lib/anystyle/document.rb

@@ -27,8 +27,8 @@ module AnyStyle
 
         case format.downcase
         when '.pdf'
-          meta = pdf_meta path if opts[:parse_meta]
-          info = pdf_info path if opts[:parse_info]
+          meta = pdf_meta path, **opts if opts[:parse_meta]
+          info = pdf_info path, **opts if opts[:parse_info]
           input = pdf_to_text path, **opts
         when '.ttx'
           tagged = true

data/lib/anystyle/feature/keyword.rb

@@ -9,11 +9,15 @@ module AnyStyle
           case alpha
           when /^ed(s|itors?|ited?|iteurs?)?$/i,
                /^(hg|hrsg|herausgeber)$/i,
-               /^(compilador)$/i
+               /^(compilador)$/i,
+               /編/
             :editor
+          when /著|撰/,
+            :author
           when /^trans(l(ated|ators?|ation))?$/i,
                /^übers(etz(t|ung))?$/i,
-               /^trad(uction|ucteurs?|uit)?$/i
+               /^trad(uction|ucteurs?|uit)?$/i,
+               /譯/
             :translator
           when /^(dissertation|thesis)$/i
             :thesis
@@ -21,7 +25,7 @@ module AnyStyle
             :proceedings
           when /^(Journal|Zeitschrift|Quarterly|Magazine?|Times|Rev(iew|vue)?|Bulletin|News|Week|Gazett[ea])/
             :journal
-          when /^in$/i
+          when /^in$/i, /收入/
             :in
           when /^([AaUu]nd|y|e)$/
             :and
@@ -29,21 +33,33 @@ module AnyStyle
             :etal
           when /^(pp?|pages?|S(eiten?)?|ff?)$/
             :page
-          when /^(vol(ume)?s?|iss(ue)?|n[or]?|number|fasc(icle|icule)?)$/i
+          when /^(vol(ume)?s?|iss(ue)?|n[or]?|number|fasc(icle|icule)?|suppl(ement)?)$/i
             :volume
           when /^(ser(ies?)?|reihe|[ck]oll(e[ck]tion))$/i
             :series
+          when /^patent$/i
+            :patent
+          when /^report$/i
+            :report
           when /^(edn|edition|expanded|rev(ised)?|p?reprint(ed)?|illustrated)$/i,
+            /^editio|aucta$/i
             /^(aufl(age)?|\p{Alpha}*ausg(abe)?)$/i
             :edition
           when /^(nd|date|spring|s[uo]mmer|autumn|fall|winter|frühling|herbst)$/i,
                /^(jan(uary?)?|feb(ruary?)?|mar(ch|z)?|apr(il)?|ma[yi]|jun[ei]?)$/,
-               /^(jul[yi]?|aug(ust)?|sep(tember)?|o[ck]t(ober)?|nov(ember)?|de[cz](ember)?)$/i
+               /^(jul[yi]?|aug(ust)?|sep(tember)?|o[ck]t(ober)?|nov(ember)?|de[cz](ember)?)$/i,
+               /年/
             :date
-          when /^(pmid|pmcid|arxiv|doi|url)/i
+          when /^(doi|url)/i
             :locator
-          when /^(retrieved|accessed)$/i
+          when /^(pmid|pmcid)/i
+            :pubmed
+          when /^(arxiv)/i
+            :arxiv
+          when /^(retrieved|retirado|accessed)$/i
             :accessed
+          when /^[ILXVMCD]{2,}$/
+            :roman
           else
             :none
           end

data/lib/anystyle/feature/line.rb

@@ -32,7 +32,7 @@ module AnyStyle
           :list
         when /^(\p{Lu}\.?)\s*(\d+\.)+\s+\p{L}+/
           :title
-        when /^(\w+\s)?(tab(le|elle|\.)|fig(ure|\.))/i
+        when /^(\w+\s)?(tab(le|elle|\.)|fig(ure|\.)|equation|graph|abb(ildung)?)/i
           :cap
         when /^\p{Pd}?\d+\p{Pd}?$/, /^[ivx]+$/i
           :num

data/lib/anystyle/feature/quotes.rb

@@ -3,13 +3,13 @@ module AnyStyle
     class Quotes < Feature
       def observe(token, **opts)
         case token
-        when /^[^"'”„’‚´«「『‘“`»」』]+$/
+        when /^[^"'”„’‚´«「『〈《‘“`»」』〉》]+$/
           :none
-        when /^["'”„’‚´«「『‘“`»].*["'”„’‚´«‘“`»」』][,;:\p{Pd}!\?\.]?$/
+        when /^["'”„’‚´«「『〈《‘“`»].*["'”„’‚´«‘“`»」』〉》][,;:\p{Pd}!\?\.]?$/
           :'quote-unquote'
-        when /^["'”„’‚´«「『‘“`»]/
+        when /^["'”„’‚´«「『‘〈《“`»]/
           :quote
-        when /["'”„’‚´«‘“`»」』][,;:\p{Pd}!\?\.]?$/
+        when /["'”„’‚´«‘“`»」』〉》][,;:\p{Pd}!\?\.]?$/
           :unquote
         else
           :other

data/lib/anystyle/feature/words.rb

@@ -14,11 +14,15 @@ module AnyStyle
         contents
         figures
         introduction
+        kurzfassung
         literatur
         literature
         references
         referenzen
+        secondary
         section
+        sources
+        summary
         tables
         works
       }

data/lib/anystyle/finder.rb

@@ -8,7 +8,10 @@ module AnyStyle
       compact: true,
       threads: 4,
       format: :references,
-      training_data: Dir[File.join(RES, 'finder', '*.ttx')].map(&:untaint)
+      training_data: Dir[File.join(RES, 'finder', '*.ttx')].map(&:untaint),
+      layout: true,
+      pdftotext: 'pdftotext',
+      pdfinfo: 'pdfinfo'
     }
 
     def initialize(options = {})
@@ -71,12 +74,18 @@ module AnyStyle
       })
     end
 
-    def prepare(input, layout: true, crop: false, **opts)
+    def prepare(input,
+                layout: options[:layout],
+                crop: false,
+                pdftotext: options[:pdftotext],
+                pdfinfo: options[:pdfinfo],
+                **opts)
+      doc_opts = { layout: layout, crop: crop, pdftotext: pdftotext, pdfinfo: pdfinfo, **opts }
       case input
       when String
-        super(Document.open(input, layout: layout, crop: crop, **opts), **opts)
+        super(Document.open(input, **doc_opts), **opts)
       when Array
-        super(Wapiti::Dataset.new(input.map { |f| Document.open(f, **opts) }), **opts)
+        super(Wapiti::Dataset.new(input.map { |f| Document.open(f, **doc_opts) }), **opts)
       else
         super(input, **opts)
       end

data/lib/anystyle/format/csl.rb

@@ -3,6 +3,7 @@ module AnyStyle
     module CSL
       def format_csl(dataset, **opts)
         format_hash(dataset).map do |hash|
+          dates_to_citeproc(hash, **opts) if hash.key?(:date)
           flatten_values hash, skip: Normalizer::Names.keys
 
           rename_value hash, :pages, :page
@@ -23,6 +24,22 @@ module AnyStyle
       end
 
       alias_method :format_citeproc, :format_csl
+
+      def dates_to_citeproc(hash, date_format: 'edtf', **opts)
+        date, = *hash.delete(:date)
+
+        case date_format.to_s
+        when 'citeproc'
+          hash[:issued] = begin
+            require 'citeproc'
+            ::CiteProc::Date.parse!(date.tr('X~', 'u?')).to_citeproc.symbolize_keys
+          rescue
+            date
+          end
+        else
+          hash[:issued] = date
+        end
+      end
     end
   end
 end

data/lib/anystyle/normalizer/arxiv.rb

@@ -0,0 +1,15 @@
+module AnyStyle
+  class Normalizer
+    class ArXiv < Normalizer
+      @keys = [:note]
+
+      def normalize(item, **opts)
+        each_value(item) do |_, value|
+          if (value =~ /arxiv:?\s*(\d{4}\.\d+(?:v\d+)?|\w+(?:.\w+)?\/\d+)/i)
+            append item, :arxiv, $1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/container.rb

@@ -5,7 +5,11 @@ module AnyStyle
 
       def normalize(item, **opts)
         map_values(item) do |_, value|
-          value.sub(/^[Ii]n(?::|\s+the)?\s+(\p{^Ll})/, '\1')
+          value
+            .sub(/^[Ii]n(?::|\s+the)?\s+(\p{^Ll})/, '\1')
+            .sub(/^of\s+/, '')
+            .sub(/^收入/, '')
+            .sub(/^(\w+ )?presented at (the )?/i, '')
         end
       end
     end

data/lib/anystyle/normalizer/locator.rb

@@ -3,7 +3,7 @@ module AnyStyle
 
   class Normalizer
     class Locator < Normalizer
-      @keys = [:isbn, :url]
+      @keys = [:isbn, :url, :doi]
 
       def normalize(item, **opts)
         map_values(item) do |key, value|
@@ -11,12 +11,20 @@ module AnyStyle
           when :isbn
             value[/[\d-]+/]
           when :url
+            doi = doi_extract(value)
+            append item, :doi, doi unless doi.nil?
             URI.extract(value)
+          when :doi
+            doi_extract(value) || value
           else
             value
           end
         end
       end
     end
+
+    def doi_extract(value)
+      value[/10\.(\d{4,9}\/[-._;()\/:A-Z0-9]+|1002\/\S+)/i]
+    end
   end
 end

data/lib/anystyle/normalizer/names.rb

@@ -30,7 +30,7 @@ module AnyStyle
             begin
               parse(strip(value))
             rescue
-              [{ literal: value }]
+              [{ literal: value.strip }]
             end
           end
         end

data/lib/anystyle/normalizer/punctuation.rb

@@ -14,8 +14,10 @@ module AnyStyle
 
       def normalize(item, **opts)
         each_value(item) do |_, value|
-          value.gsub!(/[\)\]\.,:;\p{Pd}\p{Z}\p{C}]+$/, '')
-          value.gsub!(/^[\(\[]/, '')
+          value.gsub!(/\s*[\)\]\.,:;\p{Pd}\p{Z}\p{C}。、》〉]+$/, '')
+          value.gsub!(/[,:;》〉]+$/, '')
+          value.gsub!(/^[\(\[《〈]/, '')
+          value.gsub!(/<\/?(italic|bold)>/, '')
         end
       end
     end

data/lib/anystyle/parser.rb

@@ -100,7 +100,7 @@ module AnyStyle
       compact: true,
       threads: 4,
       separator: /(?:\r?\n)+/,
-      delimiter: /\s+/,
+      delimiter: /\s+|([\uFF01-\uFF64]|。|、)/,
       format: :hash,
       training_data: File.join(RES, 'parser', 'core.xml')
     }
@@ -139,6 +139,7 @@ module AnyStyle
         Normalizer::Locator.new,
         Normalizer::Publisher.new,
         Normalizer::PubMed.new,
+        Normalizer::ArXiv.new,
         Normalizer::Names.new,
         Normalizer::Locale.new,
         Normalizer::Type.new

data/lib/anystyle/refs.rb

@@ -90,7 +90,7 @@ module AnyStyle
         indent_score(indent),
         delta_score(delta),
         years_score(a, b),
-        terminal_score(a),
+        terminal_score(a, b),
         initial_score(a, b),
         length_score(a, b),
         pages_score(a, b)
@@ -100,7 +100,7 @@ module AnyStyle
 
     def indent_score(indent)
       case
-      when indent > 0 then 1
+      when indent > 0 then 1.25
       when indent < 0 then -1
       else
         0
@@ -119,10 +119,19 @@ module AnyStyle
 
     def years_score(a, b)
       if match_year?(a)
-        if b.length > 35 && match_year?(b)
-          -1
+        if match_year?(b)
+          case
+          when b.length < 18
+            1
+          when b.length < 25
+            0.5
+          when b.length > 60
+            -0.75
+          else
+            0
+          end
         else
-          if a.match(/[\d,] (1[4-9]|2[01])\d\d[a-z]?\.$/)
+          if a.match(/[\d,] \(?(1[4-9]|2[01])\d\d[a-z]?\)?\.$/)
             -0.5
           else
             1
@@ -138,10 +147,10 @@ module AnyStyle
     end
 
     def pages_score(a, b)
-      if match_pages?(a)
+      if match_pages?(a, true)
         -0.25
       else
-        if match_pages?(b)
+        if match_pages?(b, false)
           1
         else
           0
@@ -149,23 +158,25 @@ module AnyStyle
       end
     end
 
-    def match_pages?(string)
+    def match_pages?(string, not_years = true)
       m = string.match(/(\d+)\p{Pd}(\d+)|\bpp?\.|\d+\(\d+\)/)
       return false if m.nil?
-      return false if m[1] && match_year?(m[1]) && match_year?(m[2])
+      return false if not_years && m[1] && match_year?(m[1]) && match_year?(m[2])
       return true
     end
 
-    def terminal_score(string)
-      case string
-      when /https?:\/\/\w+/i
-        -1
-      when /[,;:&\p{Pd}]$/, /(et al|pp)\.$/
+    def terminal_score(a, b)
+      case
+      when a.match(/https?:\/\/\w+/i)
+        -0.25
+      when a.match(/[,;:&\p{Pd}]$/), a.match(/\s(et al|pp|pg)\.$/)
         2
-      when /\((1[4-9]|2[01])\d\d\)\.?$/
+      when a.match(/\((1[4-9]|2[01])\d\d\)\.?$/)
         0
-      when /(\p{^Lu}\.|\])$/
+      when a.match(/(\p{^Lu}\.|\])$/)
         -1
+      when a.match(/\d$/) && b.match(/^\p{Lu}/)
+        -0.25
       else
         0
       end
@@ -177,13 +188,15 @@ module AnyStyle
         1.5
       when a.match(/\p{L}$/) && b.match(/^\p{L}/)
         1
-      when b.match(/^["'”„’‚´«「『‘“`»]/), b.match(/^url|http/i)
+      when b.match(/^["'”„’‚´«「『‘“`»]/)
         1
+      when b.match(/^(url|doi|isbn|vol)\b/i)
+        1.5
       when b.match(/^([\p{Pd}_*][\p{Pd}_* ]+|\p{Co})/)
         -1.5
       when b.match(/^\((1[4-9]|2[01])\d\d\)/) && !a.match(/(\p{Lu}|al|others)\.$/)
         -1
-      when b.match(/^\p{Lu}\p{Ll}+,\s\p{L}/) && !a.match(/\p{L}$/)
+      when b.match(/^\p{Lu}[\p{Ll}-]+,?\s\p{Lu}/) && !a.match(/\p{L}$/)
         -0.5
       when match_list?(b)
         if match_list?(a)
@@ -191,13 +204,15 @@ module AnyStyle
         else
           -0.75
         end
+      when b.match(/^\p{L}+:/), b.match(/^\p{L}+ \d/)
+        0.5
       else
         0
       end
     end
 
     def match_list?(string)
-      string.match(/^(\d{1,3}\.\s+\p{L}|\[\p{Alnum}+\])/)
+      string.match(/^(\d{1,3}(\.\s+|\s{2,})\p{L}|\[\p{Alnum}+\])/)
     end
 
     def length_score(a, b)
@@ -205,8 +220,14 @@ module AnyStyle
       when b.length < a.length
         case
         when b.length < 10
+          2.5
+        when b.length < 15
           2
+        when b.length < 20
+          1.75
         when b.length < 25
+          1.5
+        when b.length < 30
           1
         when b.length < 50
           0.75
@@ -221,7 +242,7 @@ module AnyStyle
     end
 
     def join(a, b)
-      if a.end_with? '-'
+      if a =~ /\p{Pd}$/
         if a =~ /\p{Ll}-$/ && b =~ /^\p{Ll}/
           "#{a[0...-1]}#{b}"
         else