anystyle 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 411233e757de4873a040de0adac8a862e9a8d77fe8844128ea5279f774608f45
-  data.tar.gz: '09133619934b406c21a75447d19c7b81d672bcd61e19d6f2bb0064e6b8bda718'
+SHA1:
+  metadata.gz: 3b74980250ee2a9627b97a9f91e6c11052ddd1e0
+  data.tar.gz: 645f6371761078aa5d4b49c0905a749eda55dd3b
 SHA512:
-  metadata.gz: 164b640e6b8c79a4d19ff82da141acf6d75cf133cff56624440ca63a5803a4ba457d65090bde7cb1074b503390e99430a7aaf911bd647b0484a9d982ea6fbdb2
-  data.tar.gz: ae83ce33900704257bc7ac2fc21d278125098821bcf6577bec868779a009e96743f3623f41078255f4a6b56c6b826e5899159dc2564af0f41c65035ae238af1b
+  metadata.gz: 449c2ffca67851254e4e9a98fa5618a57928147e3fda613b908ab83bc8d4d096cec7f11bae9f7727fbcad69dfac73d6f4f1034f00cece3c30a7813bfc7c2c20e
+  data.tar.gz: c14d4637d11689eb88727baabcdc30c672642d93dad810c15b86e2197835c75360c21ccf03bbdf9da3e909e0f0777d55f9ef81218700231db8ea5504051f8699

data/HISTORY.md

@@ -1,3 +1,12 @@
+1.2.0 / 2018-08-16
+==================
+* Added check and train commands to CLI.
+* Added --no-solo and --crop flags to find command.
+* Added reference block normalizer.
+* Added script detection normalizer.
+* Improved Finder reference line joining.
+* Improved Finder model; training sets.
+
 1.1.0 / 2018-07-11
 ==================
 * Improved Parser model; training sets.

data/lib/anystyle.rb

@@ -49,6 +49,8 @@ require 'anystyle/normalizer/volume'
 require 'anystyle/format/bibtex'
 require 'anystyle/format/csl'
 
+require 'anystyle/page'
+require 'anystyle/refs'
 require 'anystyle/document'
 require 'anystyle/parser'
 require 'anystyle/finder'

data/lib/anystyle/document.rb

@@ -1,7 +1,10 @@
 module AnyStyle
   class Document < Wapiti::Sequence
+
+    REFSECT = /references|referenzen|cited|bibliogra|secondary sources|literatur/i
+
     class << self
-      include PdfUtils
+      include PDFUtils
 
       def parse(string, delimiter: /\r?\n/, tagged: false)
         current_label = ''
@@ -14,7 +17,7 @@ module AnyStyle
         })
       end
 
-      def open(path, format: File.extname(path), tagged: false, layout: true, **opts)
+      def open(path, format: File.extname(path), tagged: false, **opts)
         raise ArgumentError,
           "cannot open tainted path: '#{path}'" if path.tainted?
         raise ArgumentError,
@@ -26,7 +29,7 @@ module AnyStyle
         when '.pdf'
           meta = pdf_meta path if opts[:parse_meta]
           info = pdf_info path if opts[:parse_info]
-          input = pdf_to_text path, layout: layout
+          input = pdf_to_text path, **opts
         when '.ttx'
           tagged = true
           input = File.read(path, encoding: 'utf-8')
@@ -47,8 +50,16 @@ module AnyStyle
     attr_accessor :meta, :info, :path, :pages, :tokens
     alias_method :lines, :tokens
 
+    def line_counts
+      @line_counts ||= Hash.new(0)
+    end
+
+    def nnum_counts
+      @nnum_counts ||= Hash.new(0)
+    end
+
     def pages
-      @pages ||= Page.parse(lines)
+      @pages ||= Page.parse(lines, self)
     end
 
     def each
@@ -64,28 +75,34 @@ module AnyStyle
       end
     end
 
-    def each_section
+    def each_section(skip: ['meta'])
       if block_given?
-        current = []
+        head = []
+        body = []
+        seen_content = false
+
         lines.each do |ln|
           case ln.label
           when 'title'
-            unless current.empty?
-              yield current
-              current = []
+            if seen_content
+              yield [head, body]
+              head, body, seen_content = [ln], [], false
+            else
+              head << ln
             end
           when 'ref', 'text'
-            current << ln
+            body << ln
+            seen_content = true
           else
-            # ignore
+            body << ln unless skip.include?(ln.label)
           end
         end
-        unless current.empty?
-          yield current
+        unless head.empty?
+          yield [head, body]
         end
         self
       else
-        to_enum
+        to_enum :each_section
       end
     end
 
@@ -94,7 +111,8 @@ module AnyStyle
       doc.tokens = lines.map.with_index { |line, idx|
         Wapiti::Token.new line.value,
           label: other[idx].label.to_s,
-          observations: other[idx].observations.dup
+          observations: other[idx].observations.dup,
+          score: other[idx].score
       }
       doc
     end
@@ -126,83 +144,45 @@ module AnyStyle
       }
     end
 
-    def references(**opts)
-      bib, current, delta, indent = [], nil, 0, 0
-
-      lines.each do |ln|
-        case ln.label
-        when 'ref'
-          val = display_chars(ln.value).rstrip
-          idt = val[/^\s*/].length
-          val.lstrip!
-
-          if current.nil?
-            current, delta, indent = val, 0, idt
-          else
-            if join_refs?(current, val, delta, idt - indent)
-              current = join_refs(current, val)
-            else
-              bib << current
-              current, delta, indent = val, 0, idt
-            end
-          end
-        else
-          unless current.nil?
-            if delta > 15 || %w{ blank meta }.include?(ln.label)
-              delta += 1
-            else
-              bib << current
-              current, delta, indent = nil, 0, idt
+    def references(normalize_blocks: false, **opts)
+      if normalize_blocks
+        each_section.inject([]) do |refs, (head, body)|
+          rc = body.count { |tk| tk.label == 'ref' }
+          unless rc == 0
+            tc = body.count { |tk| tk.label == 'text' }
+            is_ref_sect = !head.find { |tk| tk.value =~ REFSECT }.nil?
+
+            # Skip sections with few ref lines!
+            if is_ref_sect || include_references?(rc, tc)
+              Refs.normalize! body, max_win_size: is_ref_sect ? 6 : 2
+              refs.concat Refs.parse(body).to_a
             end
           end
-        end
-      end
 
-      unless current.nil?
-        bib << current
+          refs
+        end
+      else
+        Refs.parse(lines).to_a
       end
-
-      bib
     end
 
-    def join_refs?(a, b, delta = 0, indent = 0)
-      pro = [
-        indent > 0,
-        delta == 0,
-        b.length < 50,
-        a.length < 65,
-        !!a.match(/[,;:&\p{Pd}]$/),
-        !!b.match(/^\p{Ll}/) || !!a.match(/\p{L}$/) && !!b.match(/^\p{L}/)
-      ].count(true)
-
-      con = [
-        indent < 0,
-        delta > 8,
-        !!a.match(/\.\]$/),
-        a.length > 500,
-        (b.length - a.length) > 12,
-        !!b.match(/^(\p{Pd}\p{Pd}|\p{Lu}\p{Ll}+, \p{Lu}\.|\[\d)/)
-      ].count(true)
-
-      (pro - con) > 1
+    def include_references?(rc, tc)
+      rc > 10 || (rc + tc) > 20 && (rc.to_f / tc) > 0.2
     end
 
-    def join_refs(a, b)
-      if a[-1] == '-'
-        if b =~ /^\p{Ll}/
-          "#{a[0...-1]}#{b}"
-        else
-          "#{a}#{b}"
-        end
-      else
-        "#{a} #{b}"
+    def sections(delimiter: "\n", spacer: ' ', **opts)
+      each_section.map do |(head, body)|
+        {
+          title: head.map { |tk|
+            display_chars(tk.value).lstrip.unicode_normalize
+          }.join(spacer),
+          text: body.map { |tk|
+            display_chars(tk.value).unicode_normalize
+          }.join(delimiter)
+        }
       end
     end
 
-    def sections(delimiter: "\n", **opts)
-      []
-    end
-
     def title(delimiter: " ", **opts)
       lines.drop_while { |ln|
         ln.label != 'title'
@@ -214,51 +194,5 @@ module AnyStyle
     def inspect
       "#<AnyStyle::Document lines={#{size}}>"
     end
-
-
-    class Page
-      extend StringUtils
-
-      class << self
-        def parse(lines)
-          pages, current, width = [], [], 0
-
-          lines.each do |line|
-            if page_break?(line.value)
-              unless current.empty?
-                pages << new(current, width: width)
-              end
-
-              current = [line]
-              width = display_width(line.value)
-            else
-              current << line
-              width = [width, display_width(line.value)].max
-            end
-          end
-
-          unless current.empty?
-            pages << new(current, width: width)
-          end
-
-          pages
-        end
-      end
-
-      attr_accessor :lines, :width
-
-      def initialize(lines = [], width: 0)
-        @lines = lines
-        @width = width
-      end
-
-      def size
-        lines.size
-      end
-
-      def inspect
-        "#<AnyStyle::Document::Page size={#{size}} width={#{width}}>"
-      end
-    end
   end
 end

data/lib/anystyle/feature/line.rb

@@ -1,8 +1,8 @@
 module AnyStyle
   class Feature
     class Line < Feature
-      def observe(token, page:, **opts)
-        chars = display_chars(token).rstrip
+      def observe(token, page:, seq:, **opts)
+        chars = display_chars(token)
 
         lttrs = count(chars, /\p{L}/)
         upper = count(chars, /\p{Lu}/)
@@ -18,22 +18,37 @@ module AnyStyle
           ratio(white, chars.length),
           ratio(punct, chars.length),
           ratio(width, page.width),
-          classify(chars)
+          classify(chars),
+          page_ratio(seq.line_counts[chars], seq.pages.length),
+          page_ratio(seq.nnum_counts[nnum(chars)], seq.pages.length)
         ]
       end
 
       def classify(chars)
-        case chars
-        when /\.\s*\.\s*\.\s*\.|……+/
+        case chars.lstrip
+        when /\.\s*\.\s*\.\s*\.|……+/, /\p{L}\s{5,}\d+$/
           :toc
-        when /\s\s\s\d+$/
-          :num
-        when /^\s*(Table|Fig(ure|\.))/
+        when /^[\[\(]?\d+\.?[\]\)]?\s+\p{L}+/
+          :list
+        when /^(\p{Lu}\.?)\s*(\d+\.)+\s+\p{L}+/
+          :title
+        when /^(\w+\s)?(tab(le|elle|\.)|fig(ure|\.))/i
           :cap
+        when /^\p{Pd}?\d+\p{Pd}?$/, /^[ivx]+$/i
+          :num
+        when /copyright|©|rights reserved/i
+          :copyright
+        when /https?:\/\//i
+          :http
         else
           :none
         end
       end
+
+      def page_ratio(a, b)
+        r = a.to_f / b
+        r == 1 ? '=' : r > 1 ? '+' : (r * 10).round
+      end
     end
   end
 end

data/lib/anystyle/feature/ref.rb

@@ -3,8 +3,8 @@ module AnyStyle
     class Ref < Feature
       def observe(token, **opts)
         [
-          symbolize(count(token, /\b(1\d|20)\d\d\b/)),
-          symbolize(count(token, /(\d[\(:;]\d)|(\d\s*\p{Pd}+\s*\d)|\bpp?\.|\bvols?\.|\bnos?\./i)),
+          symbolize(count(token, /\b(1[4-9]|20)\d\d\b/)),
+          symbolize(count(token, /(\d[\(:;]\d)|(\d\s*\p{Pd}+\s*\d)|\bpp?\.|\bvols?\.|\b(nos?|nr|iss?|fasc)\.|n°|nº/i)),
           symbolize(count(token, /\b\p{Lu}\./)),
           symbolize(count(token, /\b(eds?\.|edited by|editors?|hg|hrsg|et al)\b/i)),
           token =~ /^\s*(\[\w+\]|\(\d+\)|\d+\.)\s+/ ? 'T' : 'F'
@@ -13,8 +13,8 @@ module AnyStyle
 
       def symbolize(k)
         return '-' if k < 1
-        return '+' if k < 3
-        return '++'
+        return '+' if k < 2
+        return '*'
       end
     end
   end

data/lib/anystyle/finder.rb

@@ -63,18 +63,18 @@ module AnyStyle
       dataset.map { |doc| doc.references(**opts) }
     end
 
-    def label(input, layout: true, **opts)
-      dataset = prepare(input, layout: layout, **opts)
+    def label(input, layout: true, crop: false, **opts)
+      dataset = prepare(input, layout: layout, crop: crop, **opts)
       output = model.label(dataset, **opts)
       Wapiti::Dataset.new(dataset.map.with_index { |doc, idx|
         doc.label(output[idx])
       })
     end
 
-    def prepare(input, layout: true, **opts)
+    def prepare(input, layout: true, crop: false, **opts)
       case input
       when String
-        super(Document.open(input, layout: layout, **opts), **opts)
+        super(Document.open(input, layout: layout, crop: false, **opts), **opts)
       when Array
         super(Wapiti::Dataset.new(input.map { |f| Document.open(f, **opts) }), **opts)
       else

data/lib/anystyle/normalizer/locale.rb

@@ -1,5 +1,6 @@
 module AnyStyle
   maybe_require 'language_detector'
+  maybe_require 'unicode/scripts'
 
   class Normalizer
     class Locale < Normalizer
@@ -8,23 +9,33 @@ module AnyStyle
       end
 
       def normalize(item, **opts)
-        return item if @ld.nil? || item.key?(:language)
-
         sample = item.values_at(
           :title,
           :'container-title',
-#          :'collection-title',
+          :'collection-title',
           :location,
           :journal,
-          :publisher
-#          :note
+          :publisher,
+          :note
         ).flatten.compact.join(' ')
 
         return item if sample.empty?
 
-        item[:language] = @ld.detect(sample)
+        language = detect_language(sample)
+        scripts = detect_scripts(sample)
+
+        item[:language] ||= language unless language.nil?
+        item[:scripts] ||= scripts unless scripts.nil?
         item
       end
     end
+
+    def detect_language(string)
+      @ld.detect(string) unless @ld.nil?
+    end
+
+    def detect_scripts(string)
+      ::Unicode::Scripts.scripts(string) if defined?(::Unicode::Scripts)
+    end
   end
 end

data/lib/anystyle/normalizer/names.rb

@@ -37,7 +37,7 @@ module AnyStyle
       end
 
       def repeater?(value)
-        value =~ /^[\p{Pd}_*][\p{Pd}_* ]+(,|:|\.|$)/
+        value =~ /^([\p{Pd}_*][\p{Pd}_* ]+|\p{Co})(,|:|\.|$)/
       end
 
       def strip(value)