proiel 1.1.0 → 1.3.1

data/lib/proiel/dictionary/builder.rb

@@ -0,0 +1,224 @@
+#--
+# Copyright (c) 2016-2018 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+
+# Methods for synthesising and manipulating dictionaries from treebank data.
+module PROIEL
+  class DictionaryBuilder
+    attr_reader :license
+    attr_reader :language
+    attr_reader :sources
+    attr_reader :lemmata
+
+    def initialize
+      @language = nil
+      @license = nil
+      @sources = []
+      @lemmata = {}
+      @valency = PROIEL::Valency::Lexicon.new
+    end
+
+    def add_source!(source)
+      raise ArgumentError, 'source expected' unless source.is_a?(PROIEL::Source)
+      raise ArgumentError, 'incompatible language' unless @language.nil? or @language == source.language
+      raise ArgumentError, 'incompatible license' unless @license.nil? or @license == source.license
+
+      @language ||= source.language
+      @license ||= source.license
+      @sources << source
+
+      source.tokens.each { |token| index_token!(token) }
+
+      index_homographs!
+    end
+
+    CURRENT_SCHEMA_VERSION = '3.0'.freeze
+
+    def to_xml(io)
+      builder = ::Builder::XmlMarkup.new(target: io, indent: 2)
+      builder.instruct! :xml, version: '1.0', encoding: 'UTF-8'
+      builder.proiel('export-time': DateTime.now.xmlschema, 'schema-version': CURRENT_SCHEMA_VERSION) do
+        builder.dictionary(language: @language) do
+          builder.sources do
+            @sources.each do |source|
+              builder.source(idref: source.id, license: source.license)
+            end
+          end
+
+          builder.lemmata do
+            @lemmata.sort_by { |lemma, _| lemma.downcase }.each do |form_and_pos, data|
+              form, _ = form_and_pos.split(',')
+              lemma_to_xml(builder, form, data)
+            end
+          end
+        end
+      end
+    end
+
+    def add_external_glosses!(filename, languages = %i(eng))
+      raise ArgumentError, 'filename expected' unless filename.is_a?(String)
+      raise ArgumentError, 'file not found' unless File.exists?(filename)
+
+      CSV.foreach(filename, headers: true, encoding: 'utf-8', col_sep: "\t",
+                            header_converters: :symbol, quote_char: "\b") do |row|
+        h = row.to_h
+        data = languages.map { |l| [l, h[l]] }.to_h
+
+        lemma = initialize_lemma!(row[:lemma], row[:part_of_speech])
+        lemma[:glosses] ||= {}
+        lemma[:glosses].merge!(data)
+      end
+    end
+
+    private
+
+    def initialize_lemma!(lemma, part_of_speech)
+      encoded_lemma = [lemma, part_of_speech].join(',')
+
+      @lemmata[encoded_lemma] ||= {}
+      @lemmata[encoded_lemma][:lemma] ||= lemma
+      @lemmata[encoded_lemma][:part_of_speech] ||= part_of_speech
+      @lemmata[encoded_lemma][:homographs] ||= []
+      @lemmata[encoded_lemma][:n] ||= 0
+
+      %i(distribution glosses paradigm valency).each do |k|
+        @lemmata[encoded_lemma][k] ||= {}
+      end
+
+      @lemmata[encoded_lemma]
+    end
+
+    def lemma_to_xml(builder, form, data)
+      builder.lemma(lemma: form, "part-of-speech": data[:part_of_speech]) do
+        distribution_to_xml(builder, data)
+        glosses_to_xml(builder, data)
+        homographs_to_xml(builder, data)
+        paradigm_to_xml(builder, data)
+        valency_to_xml(builder, data)
+      end
+    end
+
+    def distribution_to_xml(builder, data)
+      unless data[:distribution].empty?
+        builder.distribution do
+          data[:distribution].sort_by(&:first).each do |source_id, n|
+            builder.source(idref: source_id, n: n)
+          end
+        end
+      end
+    end
+
+    def glosses_to_xml(builder, data)
+      unless data[:glosses].empty?
+        builder.glosses do
+          data[:glosses].each do |language, value|
+            builder.gloss(value, language: language)
+          end
+        end
+      end
+    end
+
+    def homographs_to_xml(builder, data)
+      if data[:homographs].count > 0
+        builder.homographs do
+          data[:homographs].each do |homograph|
+            lemma, part_of_speech = homograph.split(',')
+            builder.homograph lemma: lemma, "part-of-speech": part_of_speech
+          end
+        end
+      end
+    end
+
+    def paradigm_to_xml(builder, data)
+      unless data[:paradigm].empty?
+        builder.paradigm do
+          data[:paradigm].sort_by(&:first).each do |morphology, d|
+            builder.slot1 morphology: morphology do
+              d.sort_by(&:first).each do |form, n|
+                builder.slot2 form: form, n: n
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def valency_to_xml(builder, data)
+      unless data[:valency].empty?
+        builder.valency do
+          frames =
+            data[:valency].map do |arguments, token_ids|
+              { arguments: arguments, tokens: token_ids }
+            end
+
+          PROIEL::Valency::Obliqueness.sort_frames(frames).each do |frame|
+            builder.frame do
+              builder.arguments do
+                frame[:arguments].each do |argument|
+                  # FIXME: deal with in a better way
+                  argument[:"part-of-speech"] = argument[:part_of_speech] if argument[:part_of_speech]
+                  argument.delete(:part_of_speech)
+                  builder.argument argument
+                end
+              end
+
+              if frame[:tokens][:a].count > 0 or frame[:tokens][:r].count > 0
+                builder.tokens do
+                  frame[:tokens][:a].each do |token_id|
+                    builder.token(flags: 'a', idref: token_id)
+                  end
+
+                  frame[:tokens][:r].each do |token_id|
+                    builder.token(flags: 'r', idref: token_id)
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def index_homographs!
+      @lemmata.keys.group_by { |l| l.split(/[,#]/).first }.each do |_, homographs|
+        if homographs.count > 1
+          homographs.each do |form|
+            @lemmata[form][:homographs] = homographs.reject { |homograph| homograph == form }
+          end
+        end
+      end
+    end
+
+    def index_token!(token)
+      if token.lemma and token.part_of_speech
+        lemma = initialize_lemma!(token.lemma, token.part_of_speech)
+
+        lemma[:n] += 1
+
+        lemma[:distribution][token.source.id] ||= 0
+        lemma[:distribution][token.source.id] += 1
+
+        lemma[:paradigm][token.morphology] ||= {}
+        lemma[:paradigm][token.morphology][token.form] ||= 0
+        lemma[:paradigm][token.morphology][token.form] += 1
+
+        # Find verbal nodes
+        if token.part_of_speech[/^V/]
+          frame = PROIEL::Valency::Arguments.get_argument_frame(token)
+
+          lemma[:valency][frame] ||= { a: [], r: [] }
+
+          entry = lemma[:valency][frame]
+
+          if token.dependents.any? { |d| d.relation == 'aux' and d.part_of_speech == 'Pk' }
+            entry[:r] << token.id
+          else
+            entry[:a] << token.id
+          end
+        end
+      end
+    end
+  end
+end

data/lib/proiel/div.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2015-2016 Marius L. Jøhndal
+# Copyright (c) 2015-2017 Marius L. Jøhndal
 #
 # See LICENSE in the top-level source directory for licensing terms.
 #++
@@ -89,10 +89,13 @@ module PROIEL
     # Returns the printable form of the div with all token forms and any
     # presentation data.
     #
+    # @param custom_token_formatter [Lambda] formatting function for tokens
+    # which is passed the token as its sole argument
+    #
     # @return [String] the printable form of the div
-    def printable_form(options = {})
+    def printable_form(custom_token_formatter: nil)
       [presentation_before,
-       @children.map { |s| s.printable_form(options) },
+       @children.map { |s| s.printable_form(custom_token_formatter: custom_token_formatter) },
        presentation_after].compact.join
     end
 
@@ -135,5 +138,21 @@ module PROIEL
         end
       end
     end
+
+    # Returns the aligned div if any.
+    #
+    # @return [Div, NilClass] aligned div
+    def alignment(aligned_source)
+      alignment_id ? aligned_source.treebank.find_div(alignment_id) : nil
+    end
+
+    # Returns inferred aligned divs if any.
+    #
+    # @return [Array<Div>] inferred aligned divs
+    def inferred_alignment(aligned_source)
+      sentences.map do |sentence|
+        sentence.inferred_alignment(aligned_source)
+      end.flatten.compact.map(&:div).uniq
+    end
   end
 end

data/lib/proiel/language.rb

@@ -0,0 +1,108 @@
+#--
+# Copyright (c) 2019 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+module PROIEL
+  module Language
+    SUPPORTED_LANGUAGES = {
+      # This is a subset of language codes from ISO 639-3 and Glottolog.
+      ang: 'Old English (ca. 450-1100)',
+      ave: 'Avestan',
+      axm: 'Middle Armenian',
+      chu: 'Church Slavic',
+      cms: 'Messapic',
+      cnx: 'Middle Cornish',
+      dum: 'Middle Dutch',
+      enm: 'Middle English',
+      frk: 'Old Frankish',
+      frm: 'Middle French',
+      fro: 'Old French (842-ca. 1400)',
+      ghc: 'Hiberno-Scottish Gaelic',
+      gmh: 'Middle High German',
+      gml: 'Middle Low German',
+      gmy: 'Mycenaean Greek',
+      goh: 'Old High German (ca. 750-1050)',
+      got: 'Gothic',
+      grc: 'Ancient Greek (to 1453)',
+      hit: 'Hittite',
+      hlu: 'Hieroglyphic Luwian',
+      htx: 'Middle Hittite',
+      lat: 'Latin',
+      lng: 'Langobardic',
+      mga: 'Middle Irish (10-12th century)',
+      non: 'Old Norse',
+      nrp: 'North Picene',
+      obt: 'Old Breton',
+      oco: 'Old Cornish',
+      odt: 'Old Dutch-Old Frankish',
+      ofs: 'Old Frisian',
+      oht: 'Old Hittite',
+      olt: 'Old Lithuanian',
+      orv: 'Old Russian',
+      osc: 'Oscan',
+      osp: 'Old Spanish',
+      osx: 'Old Saxon',
+      owl: 'Old-Middle Welsh',
+      peo: 'Old Persian (ca. 600-400 B.C.)',
+      pka: 'Ardhamāgadhī Prākrit',
+      pmh: 'Maharastri Prakrit',
+      por: 'Portuguese',
+      pro: 'Old Provençal',
+      psu: 'Sauraseni Prakrit',
+      rus: 'Russian',
+      san: 'Sanskrit',
+      sga: 'Early Irish',
+      sog: 'Sogdian',
+      spa: 'Spanish',
+      spx: 'South Picene',
+      txb: 'Tokharian B',
+      txh: 'Thracian',
+      wlm: 'Middle Welsh',
+      xbm: 'Middle Breton',
+      xcb: 'Cumbric',
+      xce: 'Celtiberian',
+      xcg: 'Cisalpine Gaulish',
+      xcl: 'Classical Armenian',
+      xum: 'Umbrian',
+      xve: 'Venetic',
+    }.freeze
+
+    # Checks if a language is supported.
+    #
+    # @param language_tag [String, Symbol] language tag of language to check
+    #
+    # @return [Boolean]
+    #
+    # @example
+    #   language_supported?(:lat)      # => true
+    #   language_supported?('grc')     # => true
+    def self.language_supported?(language_tag)
+      raise ArgumentError unless language_tag.is_a?(Symbol) or language_tag.is_a?(String)
+
+      SUPPORTED_LANGUAGES.key?(language_tag.to_sym)
+    end
+
+    # Returns the display name for a language.
+    #
+    # @param language_tag [String, Symbol] language tag of language
+    #
+    # @return [String]
+    #
+    # @example
+    #   get_display_name(:lat)         # => "Latin"
+    def self.get_display_name(language_tag)
+      raise ArgumentError unless language_tag.is_a?(Symbol) or language_tag.is_a?(String)
+      raise ArgumentError, 'unsupported language' unless language_supported?(language_tag)
+
+      SUPPORTED_LANGUAGES[language_tag.to_sym]
+    end
+
+    # Returns tag of all supported languages
+    #
+    # @return [Array<Symbol>]
+    def self.supported_language_tags
+      SUPPORTED_LANGUAGES.keys
+    end
+  end
+end

data/lib/proiel/lemma.rb

@@ -0,0 +1,77 @@
+#--
+# Copyright (c) 2018 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+module PROIEL
+  class Lemma < TreebankObject
+    # @return [Dictionary] source that the lemma belongs to
+    attr_reader :dictionary
+
+    attr_reader :n
+
+    # @return [Hash{String, Integer}] distribution of lemmata in sources. The
+    # keys are IDs of sources, the values give the frequency of the lemma per
+    # source.
+    attr_reader :distribution
+
+    # @return [Array<[String, String]> identified homographs of this lemma. The
+    # array contains pairs of lemma form (which will be homographs of this
+    # lemma form under the orthographic conventions of the language) and parts
+    # of speech.
+    attr_reader :homographs
+
+    # @return [Hash{Symbol, String}] glosses for the current lemma. The keys
+    # are language tags and the values the glosses.
+    attr_reader :glosses
+    attr_reader :paradigm
+    attr_reader :valency
+
+    # Creates a new lemma object.
+    def initialize(parent, xml = nil)
+      @dictionary = parent
+
+      @n = nil
+
+      @distribution = {}
+      @homographs = []
+      @glosses = {}
+      @paradigm = {}
+      @valency = []
+
+      from_xml(xml) if xml
+    end
+
+    private
+
+    def from_xml(xml)
+      @n = nullify(xml.n, :int)
+
+      @distribution = xml.distribution.map { |h| [h.idref, nullify(h.n, :int)] }.to_h
+      @glosses = xml.glosses.map { |h| [h.language.to_sym, h.gloss] }.to_h
+      @homographs = xml.homographs.map { |h| [h.lemma, h.part_of_speech] }
+      @paradigm = xml.paradigm.map { |slot1| [slot1.morphology, slot1.slot2s.map { |slot2| [slot2.form, nullify(slot2.n, :int)] }.to_h] }.to_h
+      @valency =
+        xml.valency.map do |frame|
+          {
+            arguments: frame.arguments.map { |a| { relation: a.relation, lemma: a.lemma, part_of_speech: a.part_of_speech, mood: a.mood, case: a.case } },
+            tokens: frame.tokens.map { |t| { flags: t.flags, idref: t.idref } },
+          }
+        end
+    end
+
+    def nullify(s, type = nil)
+      case s
+      when NilClass, /^\s*$/
+        nil
+      else
+        case type
+        when :int
+          s.to_i
+        else
+          s.to_s
+        end
+      end
+    end
+  end
+end