anystyle 1.0.0

data/lib/anystyle/normalizer/edition.rb

@@ -0,0 +1,16 @@
+module AnyStyle
+  class Normalizer
+    class Edition < Normalizer
+      @keys = [:edition]
+
+      def normalize(item, **opts)
+        map_values(item) do |_, value|
+          value
+            .gsub(/rev\./, 'revised')
+            .gsub(/([eé]d(\.|ition)?|ausg(\.|abe)?)$/i, '')
+            .strip
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/journal.rb

@@ -0,0 +1,14 @@
+module AnyStyle
+  class Normalizer
+    class Journal < Normalizer
+      def normalize(item, **opts)
+        if item.key?(:journal)
+          item[:type] = 'article-journal'
+          item[:journal].each { |journal| append item, :'container-title', journal }
+          item.delete(:journal)
+        end
+        item
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/locale.rb

@@ -0,0 +1,30 @@
+module AnyStyle
+  maybe_require 'language_detector'
+
+  class Normalizer
+    class Locale < Normalizer
+      def initialize
+        @ld = LanguageDetector.new if defined?(LanguageDetector)
+      end
+
+      def normalize(item, **opts)
+        return item if @ld.nil? || item.key?(:language)
+
+        sample = item.values_at(
+          :title,
+          :'container-title',
+#          :'collection-title',
+          :location,
+          :journal,
+          :publisher
+#          :note
+        ).flatten.compact.join(' ')
+
+        return item if sample.empty?
+
+        item[:language] = @ld.detect(sample)
+        item
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/location.rb

@@ -0,0 +1,24 @@
+module AnyStyle
+  class Normalizer
+    class Location < Normalizer
+      @keys = [:location]
+
+      def normalize(item, **opts)
+        map_values(item) do |_, value|
+          location = strip value
+
+          if !item.key?(:publisher) && location.include?(':')
+            location, publisher = location.split(/\s*:\s*/)
+            item[:publisher] = publisher
+          end
+
+          location
+        end
+      end
+
+      def strip(string)
+        string.gsub(/^\p{^Alnum}+|\p{^Alnum}+$/, '')
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/locator.rb

@@ -0,0 +1,22 @@
+module AnyStyle
+  require 'uri'
+
+  class Normalizer
+    class Locator < Normalizer
+      @keys = [:isbn, :url]
+
+      def normalize(item, **opts)
+        map_values(item) do |key, value|
+          case key
+          when :isbn
+            value[/[\d-]+/]
+          when :url
+            URI.extract(value)
+          else
+            value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/names.rb

@@ -0,0 +1,88 @@
+module AnyStyle
+  require 'namae'
+
+  class Normalizer
+    class Names < Normalizer
+      @keys = [
+        :author, :editor, :translator, :director, :producer
+      ]
+
+      attr_accessor :namae
+
+      def initialize(**opts)
+        super(**opts)
+
+        @namae = Namae::Parser.new({
+          prefer_comma_as_separator: true,
+          separator: /\A(and|AND|&|;|und|UND|y|e)\s+/,
+          appellation: /\A(?!x)x/,
+          title: /\A(?!x)x/
+        })
+      end
+
+      def normalize(item, prev: [], **opts)
+        map_values(item) do |key, value|
+          value.gsub!(/(^[\(\[]|[,;:\)\]]+$)/, '')
+          case
+          when repeater?(value) && prev.length > 0
+            prev[-1][key][0] || prev[-1][:author][0]
+          else
+            begin
+              parse(strip(value))
+            rescue
+              [{ literal: value }]
+            end
+          end
+        end
+      end
+
+      def repeater?(value)
+        value =~ /^[\p{P}\s]+$/
+      end
+
+      def strip(value)
+        value
+          .gsub(/^[Ii]n:?\s+/, '')
+          .gsub(/\b[EÉeé]d(s?\.|itors?\.?|ited|iteurs?|ité)(\s+(by|par)\s+|\b|$)/, '')
+          .gsub(/\b([Hh](rsg|gg?)\.|Herausgeber)\s+/, '')
+          .gsub(/\b[Hh]erausgegeben von\s+/, '')
+          .gsub(/\b((d|ein)er )?[Üü]ber(s\.|setzt|setzung|tragen|tragung) v(\.|on)\s+/, '')
+          .gsub(/\b[Tt]rans(l?\.|lated|lation)(\s+by\b)?\s*/, '')
+          .gsub(/\b[Tt]rad(ucteurs?|(uit|\.)(\s+par\b)?)\s*/, '')
+          .gsub(/\b([Dd]ir(\.|ected))(\s+by)?\s+/, '')
+          .gsub(/\b([Pp]rod(\.|uce[rd]))(\s+by)?\s+/, '')
+          .gsub(/\b([Pp]erf(\.|orme[rd]))(\s+by)?\s+/, '')
+          .gsub(/\*/, '')
+          .gsub(/\([^\)]*\)?/, '')
+          .gsub(/\[[^\]]*\)?/, '')
+          .gsub(/[;:]/, ',')
+          .gsub(/^\p{^L}+|\s+\p{^L}+$/, '')
+          .gsub(/[\s,\.]+$/, '')
+          .gsub(/,{2,}/, ',')
+          .gsub(/\s+\./, '.')
+      end
+
+      def parse(value)
+        raise ArgumentError if value.empty?
+
+        others = value.sub!(
+          /(,\s+)?((\&\s+)?\bet\s+(al|coll)\b|\bu\.\s*a\b|(\band|\&)\s+others).*$/, ''
+        ) || value.sub!(/\.\.\.|…/, '')
+
+        # Add surname/initial punctuation separator for Vancouver-style names
+        # E.g. Rang HP, Dale MM, Ritter JM, Moore PK
+        if value.match(/^(\p{Lu}[^\s,.]+)\s+([\p{Lu}][\p{Lu}\-]{0,3})(,|[.]?$)/)
+          value.gsub!(/\b(\p{Lu}[^\s,.]+)\s+([\p{Lu}][\p{Lu}\-]{0,3})(,|[.]?$)/, '\1, \2\3')
+        end
+
+        names = namae.parse!(value).map { |name|
+          name.normalize_initials
+          name.to_h.reject { |_, v| v.nil? }
+        }
+
+        names << { others: true } unless others.nil?
+        names
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/page.rb

@@ -0,0 +1,29 @@
+module AnyStyle
+  class Normalizer
+    class Page < Normalizer
+      @keys = [:pages]
+
+      def normalize(item, **opts)
+        map_values(item) do |_, value|
+          pages = case value
+            when /(\d+)(?:\.(\d+))?(?:\((\d{4})\))?:(\d.*)/
+              # "volume.issue(year):pp"
+              append(item, :volume, $1.to_i)
+              append(item, :issue, $2.to_i) unless $2.nil?
+              append(item, :year, $3.to_i) unless $3.nil?
+              $4
+            else
+              value
+            end
+
+          # TODO chap. 5, pp. 195-234.
+
+          pages
+            .gsub(/\p{Pd}+/, '–')
+            .gsub(/[^\d,–]+/, ' ')
+            .strip
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/publisher.rb

@@ -0,0 +1,18 @@
+module AnyStyle
+  class Normalizer
+    class Publisher < Normalizer
+      @keys = [:publisher]
+
+      def normalize(item, **opts)
+        replace_author(item) if item.key?(:author)
+        item
+      end
+
+      def replace_author(item)
+        each_value(item) do |_, value|
+          value.gsub!(/^Author$/, item[:author][0])
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/pubmed.rb

@@ -0,0 +1,18 @@
+module AnyStyle
+  class Normalizer
+    class PubMed < Normalizer
+      @keys = [:note]
+
+      def normalize(item, **opts)
+        each_value(item) do |_, value|
+          if (value =~ /PMID:?\s*(\d+)/)
+            append item, :pmid, $1
+          end
+          if (value =~ /PMC(\d+)/)
+            append item, :pmcid, $1
+          end
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/punctuation.rb

@@ -0,0 +1,23 @@
+module AnyStyle
+  class Normalizer
+    class Punctuation < Normalizer
+      @keys = [
+        :'container-title',
+        :'collection-title',
+        :date,
+        :edition,
+        :journal,
+        :location,
+        :publisher,
+        :title
+      ]
+
+      def normalize(item, **opts)
+        each_value(item) do |_, value|
+          value.gsub!(/[\)\]\.,:;\p{Pd}\p{Z}\p{C}]+$/, '')
+          value.gsub!(/^[\(\[]/, '')
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/quotes.rb

@@ -0,0 +1,14 @@
+module AnyStyle
+  class Normalizer
+    class Quotes < Normalizer
+      QUOTES = /^[«‹»›„‚“‟‘‛”’"❛❜❟❝❞⹂〝〞〟\[]|[«‹»›„‚“‟‘‛”’"❛❜❟❝❞⹂〝〞〟\]]$/
+      @keys = [:title, :'citation-number', :medium]
+
+      def normalize(item, **opts)
+        each_value(item) do |_, value|
+          value.gsub! QUOTES, ''
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/type.rb

@@ -0,0 +1,54 @@
+module AnyStyle
+  class Normalizer
+    class Type < Normalizer
+      def normalize(item, **opts)
+        item[:type] = classify item unless item.key?(:type)
+        item
+      end
+
+      def classify(item)
+        keys = item.keys
+
+        case
+        when keys.include?(:'container-title')
+          case
+          when keys.include?(:issue)
+            'article-journal'
+          when item[:'container-title'].to_s =~ /proceedings|proc\.|conference|meeting|symposi(on|um)/i
+            'paper-conference'
+          when item[:'container-title'].to_s =~ /journal|zeitschrift|quarterly|review|revue/i
+            'article-journal'
+          else
+            'chapter'
+          end
+        when keys.include?(:genre)
+          case item[:genre].to_s
+          when /ph(\.\s*)?d|diss(\.|ertation)|thesis/i
+            'thesis'
+          when /rep(\.|ort)/i
+            'report'
+          when /unpublished|manuscript/i
+            'manuscript'
+          when /patent/i
+            'patent'
+          when /personal communication/i
+            'personal_communication'
+          when /interview/i
+            'interview'
+          when /web|online|en ligne/
+            'webpage'
+          end
+        when keys.include?(:medium)
+          case item[:medium].to_s
+          when /dvd|video|vhs|motion/i
+            'motion_picture'
+          when /television/i
+            'broadcast'
+          end
+        when keys.include?(:publisher)
+          'book'
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/normalizer/volume.rb

@@ -0,0 +1,26 @@
+module AnyStyle
+  class Normalizer
+    class Volume < Normalizer
+      @keys = [:volume, :pages, :date]
+
+      def normalize(item, **opts)
+        map_values(item, [:volume]) do |_, volume|
+          case volume
+          when /(\p{Lu}?\d+)\s?\(([^)]+)\)/
+            append item, :issue, $2
+            $1
+          when /(?:(\p{Lu}?\d+)[\p{P}\s]+)?(?:nos?|nr|n°|nº|iss?)\.?\s?(.+)$/i
+            volume = $1
+            append item, :issue, $2.sub(/\p{P}$/, '')
+            volume
+          else
+            volume
+              .sub(/^[\p{P}\s]+/, '')
+              .sub(/.*vol(ume)?[\p{P}\s]+/i, '')
+              .sub(/\p{P}$/, '')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/anystyle/parser.rb

@@ -0,0 +1,199 @@
+module AnyStyle
+  class ParserCore
+    include StringUtils
+
+    class << self
+      attr_reader :defaults, :formats
+
+      def load(path)
+        new :model => path
+      end
+
+      # Returns a default parser instance
+      def instance
+        Thread.current["anystyle_#{name.downcase}"] ||= new
+      end
+    end
+
+    attr_reader :model, :options, :features, :normalizers
+
+    def initialize(options = {})
+      @options = self.class.defaults.merge(options)
+      load_model
+    end
+
+    def load_model(file = options[:model])
+      unless file.nil?
+        @model = Wapiti.load(file)
+        @model.options.update_attributes options
+      else
+        @model = Wapiti::Model.new(options.reject { |k,_| k == :model })
+        @model.path = options[:model]
+      end
+
+      self
+    end
+
+    def label(input, **opts)
+      model.label prepare(input, **opts)
+    end
+
+    def check(input)
+      model.check prepare(input, tagged: true)
+    end
+
+    def train(input = options[:training_data], truncate: true)
+      load_model(nil) if truncate
+      unless input.nil? || input.empty?
+        model.train prepare(input, tagged: true)
+      end
+      model
+    end
+
+    def learn(input)
+      train(input, truncate: false)
+    end
+
+    def normalize(hash, **opts)
+      normalizers.each do |n|
+        begin
+          hash = n.normalize(hash, **opts) unless n.skip?
+        rescue => e
+          warn "Error in #{n.name} normalizer: #{e.message}"
+        end
+      end
+      hash
+    end
+
+    def expand(dataset)
+      raise NotImplementedError
+    end
+
+    def prepare(input, **opts)
+      case input
+      when Wapiti::Dataset
+        expand input
+      when Wapiti::Sequence
+        expand Wapiti::Dataset.new([input])
+      when String
+        if !input.tainted? && input.length < 1024 && File.exists?(input)
+          expand Wapiti::Dataset.open(input, opts)
+        else
+          expand Wapiti::Dataset.parse(input, opts)
+        end
+      else
+        expand Wapiti::Dataset.parse(input, opts)
+      end
+    end
+  end
+
+
+  class Parser < ParserCore
+    include Format::BibTeX
+    include Format::CSL
+
+    @formats = [:bibtex, :citeproc, :csl, :hash, :wapiti]
+
+    @defaults = {
+      model: File.join(SUPPORT, 'parser.mod'),
+      pattern: File.join(SUPPORT, 'parser.txt'),
+      compact: true,
+      threads: 4,
+      separator: /(?:\r?\n)+/,
+      delimiter: /\s+/,
+      format: :hash,
+      training_data: File.join(RES, 'parser', 'core.xml')
+    }
+
+    def initialize(options = {})
+      super(options)
+
+      @features = [
+        Feature::Canonical.new,
+        Feature::Category.new,
+        Feature::Affix.new(size: 2),
+        Feature::Affix.new(size: 2, suffix: true),
+        Feature::Caps.new,
+        Feature::Number.new,
+        Feature::Dictionary.new(dictionary: options[:dictionary] || Dictionary.instance),
+        Feature::Keyword.new,
+        Feature::Position.new,
+        Feature::Punctuation.new,
+        Feature::Brackets.new,
+        Feature::Terminal.new,
+        Feature::Locator.new
+      ]
+
+      @normalizers = [
+        Normalizer::Quotes.new,
+        Normalizer::Brackets.new,
+        Normalizer::Punctuation.new,
+        Normalizer::Journal.new,
+        Normalizer::Container.new,
+        Normalizer::Edition.new,
+        Normalizer::Volume.new,
+        Normalizer::Page.new,
+        Normalizer::Date.new,
+        Normalizer::Location.new,
+        Normalizer::Locator.new,
+        Normalizer::Publisher.new,
+        Normalizer::PubMed.new,
+        Normalizer::Names.new,
+        Normalizer::Locale.new,
+        Normalizer::Type.new
+      ]
+    end
+
+    def expand(dataset)
+      dataset.each do |seq|
+        seq.tokens.each_with_index do |tok, idx|
+          alpha = scrub tok.value
+          tok.observations = features.map { |f|
+            f.observe tok.value, alpha: alpha, idx: idx, seq: seq
+          }
+        end
+      end
+    end
+
+    def format_hash(dataset, symbolize_keys: true)
+      dataset.inject([]) { |out, seq|
+        out << normalize(seq.to_h(symbolize_keys: symbolize_keys), prev: out)
+      }
+    end
+
+    def flatten_values(hash, skip: [], spacer: ' ')
+      hash.each_pair do |key, value|
+        unless !value.is_a?(Array) || skip.include?(key)
+          if value.length > 1 && value[0].respond_to?(:join)
+            hash[key] = value.join(spacer)
+          else
+            hash[key] = value[0]
+          end
+        end
+      end
+    end
+
+    def rename_value(hash, name, new_name)
+      hash[new_name] = hash.delete name if hash.key?(name)
+    end
+
+    def parse(input, format: options[:format], **opts)
+      case format.to_sym
+      when :wapiti
+        label(input, **opts)
+      when :hash, :bibtex, :citeproc, :csl
+        formatter = "format_#{format}".to_sym
+        send(formatter, label(input, **opts), **opts)
+      else
+        raise ArgumentError, "format not supported: #{format}"
+      end
+    end
+
+    def prepare(input, **opts)
+      opts[:separator] ||= options[:separator]
+      opts[:delimiter] ||= options[:delimiter]
+      input = input.join("\n") if input.is_a?(Array) && input[0].is_a?(String)
+      super(input, opts)
+    end
+  end
+end