anystyle 1.0.0

data/lib/anystyle/feature/ref.rb

@@ -0,0 +1,21 @@
+module AnyStyle
+  class Feature
+    class Ref < Feature
+      def observe(token, **opts)
+        [
+          symbolize(count(token, /\b(1\d|20)\d\d\b/)),
+          symbolize(count(token, /(\d[\(:;]\d)|(\d\s*\p{Pd}+\s*\d)|\bpp?\.|\bvols?\.|\bnos?\./i)),
+          symbolize(count(token, /\b\p{Lu}\./)),
+          symbolize(count(token, /\b(eds?\.|edited by|editors?|hg|hrsg|et al)\b/i)),
+          token =~ /^\s*(\[\w+\]|\(\d+\)|\d+\.)\s+/ ? 'T' : 'F'
+        ]
+      end
+
+      def symbolize(k)
+        return '-' if k < 1
+        return '+' if k < 3
+        return '++'
+      end
+    end
+  end
+end

data/lib/anystyle/feature/terminal.rb

@@ -0,0 +1,19 @@
+module AnyStyle
+  class Feature
+    class Terminal < Feature
+      def observe(token, **opts)
+        case token
+        when /[\.\)\]]["'”„’‚´«‘“`»」』\)\]]?$/,
+             /,["'”„’‚´«‘“`»」』\)\]]|["'”„’‚´«‘“`»」』\)\]],$/
+          :strong
+        when /[:"'”„’‚´«‘“`»」』][,;:\p{Pd}!\?\.]?$/
+          :moderate
+        when /[!\?,;\p{Pd}]["'”„’‚´«‘“`»」』]?$/
+          :weak
+        else
+          :none
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/feature/words.rb

@@ -0,0 +1,74 @@
+module AnyStyle
+  class Feature
+    class Words < Feature
+      attr_reader :dictionary
+
+      TITLE_WORDS = %w{
+        abstract
+        acknowledgements
+        appendix
+        bibliography
+        bibliographie
+        chapter
+        cited
+        contents
+        figures
+        introduction
+        literatur
+        literature
+        references
+        referenzen
+        section
+        tables
+        works
+      }
+
+      def initialize(dictionary:, **opts)
+        super(**opts)
+        @dictionary = dictionary
+      end
+
+      def observe(token, **opts)
+        words = token.scan(/\S+/).map { |word| canonize word }.reject(&:empty?)
+        spacers = token.scan(/\S\s\s+\S/)
+        numbers = token.scan(/\d+(\.\d+)?/)
+        title = words.count { |word| TITLE_WORDS.include?(word) }
+        counts = dictionary.tag_counts(words)
+
+        if words.length > 0
+          len = words.map(&:length).sort
+          avg = len.reduce(0, :+) / len.length
+          med = len.length.even? ?
+            len[(len.length - 1) / 2, 2].reduce(0, :+) / 2 :
+            len[len.length / 2]
+        else
+          avg, med = 0, 0
+        end
+
+        [
+          words.length,
+          avg,
+          med,
+          spacers.length,
+          classify(words[0]),
+          numbers.length,
+          ratio(title, words.length),
+          *counts.map { |cnt| ratio(cnt, words.length) }
+        ]
+      end
+
+      def classify(word)
+        case word
+        when /^(\d+|[vx]?iii?|i?[vx]|)$/i
+          :number
+        when /\d/
+          :numeric
+        when nil
+          :none
+        else
+          :alpha
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/finder.rb

@@ -0,0 +1,94 @@
+module AnyStyle
+  class Finder < ParserCore
+    @formats = [:hash, :references, :wapiti]
+
+    @defaults = {
+      model: File.join(SUPPORT, 'finder.mod'),
+      pattern: File.join(SUPPORT, 'finder.txt'),
+      compact: true,
+      threads: 4,
+      format: :references,
+      training_data: Dir[File.join(RES, 'finder', '*.ttx')].map(&:untaint)
+    }
+
+    def initialize(options = {})
+      super(options)
+
+      @features = [
+        Feature::Line.new,
+        Feature::Category.new(strip: true),
+        Feature::Words.new(dictionary: options[:dictionary] || Dictionary.instance),
+        Feature::Indent.new,
+        Feature::Ref.new,
+        Feature::Position.new(seq: :page, idx: :ln),
+        Feature::Position.new(seq: :pages, idx: :pn)
+      ]
+    end
+
+    def expand(dataset)
+      dataset.each do |doc|
+        doc.each.with_index do |(line, ln, page, pn), idx|
+          line.observations = features.map.with_index { |f, fn|
+            f.observe line.value,
+              page: page,
+              pages: doc.pages,
+              seq: doc,
+              pn: pn,
+              ln: ln,
+              fn: fn,
+              idx: idx
+          }.flatten
+        end
+      end
+    end
+
+    def find(input, format: options[:format], **opts)
+      case format.to_sym
+      when :references, :ref
+        format_references(label(input, **opts), **opts)
+      when :hash
+        format_hash(label(input, **opts), **opts)
+      when :wapiti
+        label(input, **opts)
+      else
+        raise ArgumentError, "unknown format '#{format}'"
+      end
+    end
+
+    def format_hash(dataset, **opts)
+      dataset.map { |doc| doc.to_h(**opts) }
+    end
+
+    def format_references(dataset, **opts)
+      dataset.map { |doc| doc.references(**opts) }
+    end
+
+    def label(input, layout: true, **opts)
+      dataset = prepare(input, layout: layout, **opts)
+      output = model.label(dataset, **opts)
+      Wapiti::Dataset.new(dataset.map.with_index { |doc, idx|
+        doc.label(output[idx])
+      })
+    end
+
+    def prepare(input, layout: true, **opts)
+      case input
+      when String
+        super(Document.open(input, layout: layout, **opts), **opts)
+      when Array
+        super(Wapiti::Dataset.new(input.map { |f| Document.open(f, **opts) }), **opts)
+      else
+        super(input, **opts)
+      end
+    end
+
+    def save_each(dataset, dir: '.', tagged: false, **opts)
+      dataset.each.with_index do |doc, idx|
+        name = doc.path.nil? ? idx : File.basename(doc.path, File.extname(doc.path))
+        file = "#{name}.#{tagged ? 'ttx' : 'txt'}"
+        File.write(File.join(dir, file), doc.to_s(tagged: tagged, **opts))
+      end
+    end
+
+  end
+end

data/lib/anystyle/format/bibtex.rb

@@ -0,0 +1,63 @@
+module AnyStyle
+  module Format
+    module BibTeX
+      TYPES = {
+        'article-journal' => 'article',
+        'chapter' => 'incollection',
+        'manuscript' => 'unpublished',
+        'paper-conference' => 'inproceedings',
+        'report' => 'techreport'
+      }
+
+      def format_bibtex(dataset, **opts)
+        require 'bibtex'
+
+        b = ::BibTeX::Bibliography.new
+        format_hash(dataset).each do |hash|
+          flatten_values hash, skip: Normalizer::Names.keys
+
+          hash[:bibtex_type] = TYPES[hash[:type]] || hash[:type] || 'misc'
+          hash.delete :type
+
+          case hash[:bibtex_type]
+          when 'article'
+            rename_value hash, :'container-title', :journal
+            rename_value hash, :issue, :number
+          when 'techreport'
+            rename_value hash, :publisher, :institution
+          when 'thesis'
+            rename_value hash, :publisher, :school
+          end
+
+          Normalizer::Names.keys.each do |role|
+            names_to_bibtex hash, role
+          end
+
+          rename_value hash, :'collection-title', :series
+          rename_value hash, :'container-title', :booktitle
+          rename_value hash, :accessed, :urldate
+          rename_value hash, :genre, :type
+          rename_value hash, :location, :address
+
+          b << ::BibTeX::Entry.new(hash)
+        end
+        b
+      end
+
+      def names_to_bibtex(hash, role)
+        if hash.key?(role)
+          hash[role] = hash[role].map { |name|
+            case
+            when name.key?(:literal)
+              name[:literal]
+            when name.key?(:family) || name.key?(:given)
+              name.values_at(:family, :suffix, :given).compact.join(', ')
+            else
+              nil
+            end
+          }.compact.join(' and ')
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/format/csl.rb

@@ -0,0 +1,28 @@
+module AnyStyle
+  module Format
+    module CSL
+      def format_csl(dataset, **opts)
+        format_hash(dataset).map do |hash|
+          flatten_values hash, skip: Normalizer::Names.keys
+
+          rename_value hash, :pages, :page
+          rename_value hash, :location, :'publisher-place'
+          rename_value hash, :url, :URL
+          rename_value hash, :doi, :DOI
+          rename_value hash, :pmid, :PMID
+          rename_value hash, :pmcid, :PMCID
+
+          Normalizer::Names.keys.each do |role|
+            if hash.key?(role)
+              hash[role].reject! { |name| name[:others] }
+            end
+          end
+
+          hash
+        end
+      end
+
+      alias_method :format_citeproc, :format_csl
+    end
+  end
+end

data/lib/anystyle/normalizer.rb

@@ -0,0 +1,65 @@
+module AnyStyle
+  class Normalizer
+    @keys = []
+
+    class << self
+      attr_reader :keys
+    end
+
+    attr_reader :keys
+    attr_accessor :skip
+
+    def initialize(keys: self.class.keys)
+      @keys = keys
+      @skip = false
+    end
+
+    def name
+      self.class.name
+    end
+
+    def normalize(item, **opts)
+      raise NotImplementedError
+    end
+
+    def append(item, key, value)
+      if item.key?(key)
+        item[key] << value
+      else
+        item[key] = [value]
+      end
+    end
+
+    def each_value(item, keys = keys_for(item))
+      keys.each do |key|
+        item[key].each do |value|
+          yield key, value
+        end if item.key?(key)
+      end
+      item
+    end
+
+    def map_values(item, keys = keys_for(item))
+      keys.each do |key|
+        if item.key?(key)
+          item[key] = item[key].map { |value|
+            yield key, value
+          }.flatten.reject { |v| v.nil? || v.empty? }
+        end
+      end
+      item
+    end
+
+    def keys_for(item)
+      if self.class.keys.empty?
+        item.keys
+      else
+        self.class.keys
+      end
+    end
+
+    def skip?
+      @skip
+    end
+  end
+end

data/lib/anystyle/normalizer/brackets.rb

@@ -0,0 +1,13 @@
+module AnyStyle
+  class Normalizer
+    class Brackets < Normalizer
+      @keys = [:'citation-number', :note]
+
+      def normalize(item, **opts)
+        each_value(item) do |_, value|
+          value.gsub!(/^[\(\[\{]|[\]\)\}]$/, '')
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/container.rb

@@ -0,0 +1,13 @@
+module AnyStyle
+  class Normalizer
+    class Container < Normalizer
+      @keys = [:'container-title']
+
+      def normalize(item, **opts)
+        map_values(item) do |_, value|
+          value.sub(/^[Ii]n(?::|\s+the)?\s+(\p{^Ll})/, '\1')
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/date.rb

@@ -0,0 +1,109 @@
+module AnyStyle
+  class Normalizer
+    class Date < Normalizer
+      @keys = [:date]
+
+      def normalize(item, **opts)
+        map_values(item) do |_, value|
+          case
+          when unknown?(value)
+            'XXXX'
+          when interval?(value)
+            value
+          # TODO AD/BC
+          # TODO Seasons
+          when iso?(value)
+            value
+          else
+            year = extract_year(value)
+            unless year.nil?
+              month = extract_month_by_name(value)
+              day = extract_day(value) unless month.nil?
+              [
+                [year, month, day].compact.join('-'),
+                extract_uncertainty(value)
+              ].compact.join('')
+            else
+              value
+            end
+          end
+        end
+      end
+
+      def iso?(date)
+        date =~ /[012]\d\d\d-\d\d-\d\d/
+      end
+
+      def interval?(date)
+        date =~ /\/|\s\p{Pd}\s|(\s([12]?\d|30)\p{Pd}([12]?\d|3[01])?)/
+      end
+
+      def unknown?(date)
+        date =~ /inconnue|unknown|unbekannt|[ns]\. ?d\b|no date/i
+      end
+
+      def uncertain?(date)
+        date =~ /\?/
+      end
+
+      def approximate?(date)
+        date =~ /(\b(circa|ca\.|vers|approx))|(^[cv]\.)/i
+      end
+
+      def extract_uncertainty(date)
+        if approximate?(date)
+          uncertain?(date) ? '%' : '~'
+        else
+          uncertain?(date) ? '?' : nil
+        end
+      end
+
+      def extract_year(date)
+        if date =~ /\D?([012]\d\d\d)\D?/
+          $1
+        else
+          nil
+        end
+      end
+
+      def extract_day(date)
+        if date =~ /\b([012]?\d|3[01])\b/
+          '%02d' % $1.to_i
+        else
+          nil
+        end
+      end
+
+      def extract_month_by_name(date)
+        case date
+        when /\bjan/i
+          '01'
+        when /\bf(eb|év)/i
+          '02'
+        when /\bmar/i
+          '03'
+        when /\ba[pv]r/i
+          '04'
+        when /\bma[yi]/i
+          '05'
+        when /\bjui?n/i
+          '06'
+        when /\bjui?l/i
+          '07'
+        when /\ba(ug|oût)/i
+          '08'
+        when /\bsep/i
+          '09'
+        when /\bo[ck]t/i
+          '10'
+        when /\bnov/i
+          '11'
+        when /\bd[eé]c/i
+          '12'
+        else
+          nil
+        end
+      end
+    end
+  end
+end