proiel 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 10affa8825a31d3bcb810a5dbc41a7869c4fe7d7cb15b1c361cc8c13947d3c4a
-  data.tar.gz: 43145ff2225e521599bdc96983c295b2ccdef1a9b642849523f3852fb68b4d8d
+  metadata.gz: ccdb00c28a352d6f6481a76b5adf4bef5a426e98738c3ed4241134e202302aef
+  data.tar.gz: 299fde59d6c773a9f1246263f66ab3d37b4216f0b1dd873a552eb4c8d1cd6ef7
 SHA512:
-  metadata.gz: cc4b7b78021b97304c93429bab8fbe44f38a2e4740c280c5085a86ecb6c43a4e44c55936a0192196d5b769a3f54169ff8dfe64eb31305c07abd791d1e6ea0a17
-  data.tar.gz: cfcadba2ef52a4d81c6aa432549618c5c9dfef55876ae313f7cdd15704a825cb82be06b1fda0f53ef5983f17470aa443bf5be1d70d659fb066b1a3bbd57ea309
+  metadata.gz: 105c8c89b0d3df2491fb51a03dbf96797af8e195edcdb1b901b12e81e0a632dac1e8b2ae6b398606fbc0b18b856134c338410094a5d03efd3783a7fed6b756e1
+  data.tar.gz: ce513c17bfa2301928551a49c81147f6da693c38a733b2cc749705f5f96dc2798c6dd48729e2929a3202f1061ba458c306470aedf6598c684744dfc2b74acfd4

data/lib/proiel.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2015-2017 Marius L. Jøhndal
+# Copyright (c) 2015-2018 Marius L. Jøhndal
 #
 # See LICENSE in the top-level source directory for licensing terms.
 #++
@@ -15,6 +15,7 @@ require 'erb'
 require 'open3'
 require 'set'
 require 'builder'
+require 'csv'
 
 require 'proiel/version'
 require 'proiel/utils'
@@ -32,7 +33,12 @@ require 'proiel/source'
 require 'proiel/div'
 require 'proiel/sentence'
 require 'proiel/token'
+require 'proiel/dictionary'
+require 'proiel/dictionary/builder'
+require 'proiel/lemma'
 require 'proiel/visualization'
 require 'proiel/chronology'
 require 'proiel/valency'
-require 'proiel/dictionary'
+require 'proiel/dictionary/builder'
+require 'proiel/alignment'
+require 'proiel/language'

data/lib/proiel/alignment.rb

@@ -0,0 +1,3 @@
+module PROIEL::Alignment; end
+
+require 'proiel/alignment/builder'

data/lib/proiel/alignment/builder.rb

@@ -0,0 +1,220 @@
+module PROIEL
+  module Alignment
+    module Builder
+      # This computes a matrix of original and translation sentences that are
+      # aligned. For now, this function does not handle translation sentences that
+      # are unaligned (this is tricky to handle robustly!). As the current treebank
+      # collection stands this is an issue that *should* not arise so this is for
+      # now a reasonable approximation.
+      def self.compute_matrix(alignment, source, blacklist = [], log_directory = nil)
+        matrix1 = group_backwards(alignment, source, blacklist)
+        raise unless matrix1.map { |r| r[:original]    }.flatten.compact == alignment.sentences.map(&:id)
+
+        matrix2 = group_forwards(alignment, source, blacklist)
+        raise unless matrix2.map { |r| r[:translation] }.flatten.compact == source.sentences.map(&:id)
+
+        if log_directory
+          # Verify that both texts are still in the correct sequence
+          File.open(File.join(log_directory, "#{source.id}1"), 'w') do |f|
+            matrix1.map do |x|
+              f.puts x.inspect
+            end
+          end
+
+          File.open(File.join(log_directory, "#{source.id}2"), 'w') do |f|
+            matrix2.map do |x|
+              f.puts x.inspect
+            end
+          end
+        end
+
+        matrix = []
+        iter1 = { i: 0, m: matrix1 }
+        iter2 = { i: 0, m: matrix2 }
+
+        loop do
+          # Take from matrix1 unless we have a translation
+          while iter1[:i] < iter1[:m].length and iter1[:m][iter1[:i]][:translation].empty?
+            matrix << iter1[:m][iter1[:i]]
+            iter1[:i] += 1
+          end
+
+          # Take from matrix2 unless we have an original
+          while iter2[:i] < iter2[:m].length and iter2[:m][iter2[:i]][:original].empty?
+            matrix << iter2[:m][iter2[:i]]
+            iter2[:i] += 1
+          end
+
+          if iter1[:i] < iter1[:m].length and iter2[:i] < iter2[:m].length
+            # Now the two should match provided alignments are sorted the same way,
+            # so take one from each. If they don't match outright, we may have a case
+            # of swapped sentence orders or a gap (one sentence unaligned in one of
+            # the texts surrounded by two sentences that are aligned to the same
+            # sentence in the other text). We'll try to repair this by merging bits
+            # from the next row in various combinations.
+            #
+            # When adding to the new mateix, pick original from matrix1 and
+            # translation from matrix2 so that the original textual order is
+            # preserved
+            if repair(matrix, iter1, 0, iter2, 0) or
+
+               repair(matrix, iter1, 1, iter2, 0) or
+               repair(matrix, iter1, 0, iter2, 1) or
+               repair(matrix, iter1, 1, iter2, 1) or
+
+               repair(matrix, iter1, 2, iter2, 0) or
+               repair(matrix, iter1, 0, iter2, 2) or
+               repair(matrix, iter1, 2, iter2, 1) or
+               repair(matrix, iter1, 1, iter2, 2) or
+               repair(matrix, iter1, 2, iter2, 2) or
+
+               repair(matrix, iter1, 3, iter2, 0) or
+               repair(matrix, iter1, 0, iter2, 3) or
+               repair(matrix, iter1, 3, iter2, 1) or
+               repair(matrix, iter1, 1, iter2, 3) or
+               repair(matrix, iter1, 3, iter2, 2) or
+               repair(matrix, iter1, 2, iter2, 3) or
+               repair(matrix, iter1, 3, iter2, 3) or
+
+               repair(matrix, iter1, 4, iter2, 0) or
+               repair(matrix, iter1, 0, iter2, 4) or
+               repair(matrix, iter1, 4, iter2, 1) or
+               repair(matrix, iter1, 1, iter2, 4) or
+               repair(matrix, iter1, 4, iter2, 2) or
+               repair(matrix, iter1, 2, iter2, 4) or
+               repair(matrix, iter1, 4, iter2, 3) or
+               repair(matrix, iter1, 3, iter2, 4) or
+               repair(matrix, iter1, 4, iter2, 4)
+            else
+              STDERR.puts iter1[:i], iter1[:m][iter1[:i]].inspect
+              STDERR.puts iter2[:i], iter2[:m][iter2[:i]].inspect
+              raise
+            end
+          else
+            raise unless iter1[:i] == iter1[:m].length and iter2[:i] == iter2[:m].length
+            break
+          end
+        end
+
+        if log_directory
+          File.open(File.join(log_directory, "#{source.id}3"), 'w') do |f|
+            matrix.map do |x|
+              f.puts x.inspect
+            end
+          end
+        end
+
+        raise unless matrix.map { |r| r[:original]    }.flatten.compact == alignment.sentences.map(&:id)
+        raise unless matrix.map { |r| r[:translation] }.flatten.compact == source.sentences.map(&:id)
+
+        matrix
+      end
+
+      private
+
+      def self.group_forwards(alignment, source, blacklist = [])
+        # Make an original to translation ID mapping
+        mapping = {}
+
+        source.sentences.each do |sentence|
+          mapping[sentence.id] = []
+
+          next if blacklist.include?(sentence.id)
+
+          mapping[sentence.id] = sentence.inferred_alignment(alignment).map(&:id)
+        end
+
+        # Translate to a pairs of ID arrays, chunk original IDs that share at least
+        # one translation ID, then reduce the result so we get an array of m-to-n
+        # relations
+        mapping.map do |v, k|
+          { original: k, translation: [v] }
+        end.chunk_while do |x, y|
+          !(x[:original] & y[:original]).empty?
+        end.map do |chunk|
+          chunk.inject do |a, v|
+            a[:original] += v[:original]
+            a[:translation] += v[:translation]
+            a
+          end
+        end.map do |row|
+          { original: row[:original].uniq, translation: row[:translation] }
+        end
+      end
+
+      def self.group_backwards(alignment, source, blacklist = [])
+        # Make an original to translation ID mapping
+        mapping = {}
+
+        alignment.sentences.each do |sentence|
+          mapping[sentence.id] = []
+        end
+
+        source.sentences.each do |sentence|
+          next if blacklist.include?(sentence.id)
+
+          original_ids = sentence.inferred_alignment(alignment).map(&:id)
+
+          original_ids.each do |original_id|
+            mapping[original_id] << sentence.id
+          end
+        end
+
+        # Translate to a pairs of ID arrays, chunk original IDs that share at least
+        # one translation ID, then reduce the result so we get an array of m-to-n
+        # relations
+        mapping.map do |k, v|
+          { original: [k], translation: v }
+        end.chunk_while do |x, y|
+          !(x[:translation] & y[:translation]).empty?
+        end.map do |chunk|
+          chunk.inject do |a, v|
+            a[:original] += v[:original]
+            a[:translation] += v[:translation]
+            a
+          end
+        end.map do |row|
+          { original: row[:original], translation: row[:translation].uniq }
+        end
+      end
+
+      def self.repair_merge_cells(iter, delta, field)
+        matrix, i = iter[:m], iter[:i]
+        (0..delta).map { |j| matrix[i + j][field] }.inject(&:+)
+      end
+
+      def self.select_unaligned(iter, delta, field, check_field)
+        matrix, i = iter[:m], iter[:i]
+        (0..delta).select { |j| matrix[i + j][check_field].empty? }.map { |j| matrix[i + j][field] }.flatten
+      end
+
+      def self.repair(matrix, iter1, delta1, iter2, delta2)
+        o1 = repair_merge_cells(iter1, delta1, :original)
+        o2 = repair_merge_cells(iter2, delta2, :original)
+
+        t1 = repair_merge_cells(iter1, delta1, :translation)
+        t2 = repair_merge_cells(iter2, delta2, :translation)
+
+        u1 = select_unaligned(iter1, delta1, :original, :translation)
+        u2 = select_unaligned(iter2, delta2, :translation, :original)
+
+        if o1.sort - u1 == o2.sort.uniq and t1.sort.uniq == t2.sort - u2
+          unless delta1.zero? and delta2.zero?
+            STDERR.puts "Assuming #{delta1 + 1}/#{delta2 + 1} swapped sentence order:"
+            STDERR.puts ' * ' + (0..delta1).map { |j| iter1[:m][iter1[:i] + j].inspect }.join(' + ')
+            STDERR.puts ' * ' + (0..delta2).map { |j| iter2[:m][iter2[:i] + j].inspect }.join(' + ')
+          end
+
+          matrix << { original: o1, translation: t2 }
+
+          iter1[:i] += delta1 + 1
+          iter2[:i] += delta2 + 1
+
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/proiel/annotation_schema.rb

@@ -22,10 +22,17 @@ module PROIEL
 
     # Creates a new annotation schema object.
     def initialize(xml_object)
-      @part_of_speech_tags = make_part_of_speech_tags(xml_object).freeze
-      @relation_tags = make_relation_tags(xml_object).freeze
-      @morphology_tags = make_morphology_tags(xml_object).freeze
-      @information_status_tags = make_information_status_tags(xml_object).freeze
+      if xml_object
+        @part_of_speech_tags = make_part_of_speech_tags(xml_object).freeze
+        @relation_tags = make_relation_tags(xml_object).freeze
+        @morphology_tags = make_morphology_tags(xml_object).freeze
+        @information_status_tags = make_information_status_tags(xml_object).freeze
+      else
+        @part_of_speech_tags = {}.freeze
+        @relation_tags = {}.freeze
+        @morphology_tags = {}.freeze
+        @information_status_tags = {}.freeze
+      end
     end
 
     # @return [Hash<String,RelationTagDefinition>] definition of primary relation tags

data/lib/proiel/dictionary.rb

@@ -1,3 +1,79 @@
-module PROIEL::Dictionary; end
+#--
+# Copyright (c) 2018 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+module PROIEL
+  class Dictionary < TreebankObject
+    # @return [Treebank] treebank that this source belongs to
+    attr_reader :treebank
 
-require 'proiel/dictionary/builder'
+    # @return [String] language of the source as an ISO 639-3 language tag
+    attr_reader :language
+
+    # @return [String] dialect of the source
+    attr_reader :dialect
+
+    # @return [DateTime] export time for the dictionary
+    attr_reader :export_time
+
+    # @return [Hash] all lemmata in the dictionary
+    attr_reader :lemmata
+
+    # @return [Integer] number of lemmata in the dictionary
+    attr_reader :n
+
+    # @return [Hash] all sources in the dictionary
+    attr_reader :sources
+
+    # Creates a new dictionary object.
+    def initialize(parent, export_time, language, dialect, xml = nil)
+      @treebank = parent
+
+      raise ArgumentError, 'string or nil expected' unless export_time.nil? or export_time.is_a?(String)
+      @export_time = export_time.nil? ? nil : DateTime.parse(export_time).freeze
+
+      @language = language.freeze
+      @dialect = dialect ? dialect.freeze : nil
+
+      @lemmata = {}
+      @sources = {}
+      @n = 0
+
+      from_xml(xml) if xml
+    end
+
+    # FIXME
+    def id
+      @language
+    end
+
+    private
+
+    def from_xml(xml)
+      xml.sources.each do |s|
+        @sources[s.idref] = { license: nullify(s.license), n: nullify(s.n, :int) }
+      end
+
+      xml.lemmata.each do |l|
+        @lemmata[l.lemma] ||= {}
+        @lemmata[l.lemma][l.part_of_speech] = Lemma.new(self, l)
+        @n += 1
+      end
+    end
+
+    def nullify(s, type = nil)
+      case s
+      when NilClass, /^\s*$/
+        nil
+      else
+        case type
+        when :int
+          s.to_i
+        else
+          s.to_s
+        end
+      end
+    end
+  end
+end

data/lib/proiel/dictionary/builder.rb

@@ -1,12 +1,12 @@
 #--
-# Copyright (c) 2016-2017 Marius L. Jøhndal
+# Copyright (c) 2016-2018 Marius L. Jøhndal
 #
 # See LICENSE in the top-level source directory for licensing terms.
 #++
 
 # Methods for synthesising and manipulating dictionaries from treebank data.
-module PROIEL::Dictionary
-  class Builder
+module PROIEL
+  class DictionaryBuilder
     attr_reader :license
     attr_reader :language
     attr_reader :sources
@@ -43,12 +43,13 @@ module PROIEL::Dictionary
         builder.dictionary(language: @language) do
           builder.sources do
             @sources.each do |source|
-              builder.source(id: source.id, license: source.license)
+              builder.source(idref: source.id, license: source.license)
             end
           end
 
-          builder.lemmata(n: @lemmata.count) do
-            @lemmata.sort_by { |lemma, _| lemma.downcase }.each do |form, data|
+          builder.lemmata do
+            @lemmata.sort_by { |lemma, _| lemma.downcase }.each do |form_and_pos, data|
+              form, _ = form_and_pos.split(',')
               lemma_to_xml(builder, form, data)
             end
           end
@@ -56,10 +57,41 @@ module PROIEL::Dictionary
       end
     end
 
+    def add_external_glosses!(filename, languages = %i(eng))
+      raise ArgumentError, 'filename expected' unless filename.is_a?(String)
+      raise ArgumentError, 'file not found' unless File.exists?(filename)
+
+      CSV.foreach(filename, headers: true, encoding: 'utf-8', col_sep: "\t",
+                  header_converters: :symbol, quote_char: "\b") do |row|
+        h = row.to_h
+        data = languages.map { |l| [l, h[l]] }.to_h
+
+        lemma = initialize_lemma!(row[:lemma], row[:part_of_speech])
+        lemma[:glosses] ||= {}
+        lemma[:glosses].merge!(data)
+      end
+    end
+
     private
 
+    def initialize_lemma!(lemma, part_of_speech)
+      encoded_lemma = [lemma, part_of_speech].join(',')
+
+      @lemmata[encoded_lemma] ||= {}
+      @lemmata[encoded_lemma][:lemma] ||= lemma
+      @lemmata[encoded_lemma][:part_of_speech] ||= part_of_speech
+      @lemmata[encoded_lemma][:homographs] ||= []
+      @lemmata[encoded_lemma][:n] ||= 0
+
+      %i(distribution glosses paradigm valency).each do |k|
+        @lemmata[encoded_lemma][k] ||= {}
+      end
+
+      @lemmata[encoded_lemma]
+    end
+
     def lemma_to_xml(builder, form, data)
-      builder.lemma(form: form, part_of_speech: data[:part_of_speech], n: data[:n]) do
+      builder.lemma(lemma: form, "part-of-speech": data[:part_of_speech]) do
         distribution_to_xml(builder, data)
         glosses_to_xml(builder, data)
         homographs_to_xml(builder, data)
@@ -69,17 +101,21 @@ module PROIEL::Dictionary
     end
 
     def distribution_to_xml(builder, data)
-      builder.distribution do
-        data[:distribution].sort_by(&:first).each do |source_id, n|
-          builder.source(id: source_id, n: n)
+      unless data[:distribution].empty?
+        builder.distribution do
+          data[:distribution].sort_by(&:first).each do |source_id, n|
+            builder.source(idref: source_id, n: n)
+          end
         end
       end
     end
 
     def glosses_to_xml(builder, data)
-      if data[:glosses].count > 0
+      unless data[:glosses].empty?
         builder.glosses do
-          # TODO
+          data[:glosses].each do |language, value|
+            builder.gloss(value, language: language)
+          end
         end
       end
     end
@@ -88,7 +124,8 @@ module PROIEL::Dictionary
       if data[:homographs].count > 0
         builder.homographs do
           data[:homographs].each do |homograph|
-            builder.lemma form: homograph
+            lemma, part_of_speech = homograph.split(',')
+            builder.homograph lemma: lemma, "part-of-speech": part_of_speech
           end
         end
       end
@@ -120,22 +157,21 @@ module PROIEL::Dictionary
             builder.frame do
               builder.arguments do
                 frame[:arguments].each do |argument|
+                  # FIXME: deal with in a better way
+                  argument[:"part-of-speech"] = argument[:part_of_speech] if argument[:part_of_speech]
+                  argument.delete(:part_of_speech)
                   builder.argument argument
                 end
               end
 
-              if frame[:tokens][:a].count > 0
-                builder.tokens flags: 'a', n: frame[:tokens][:a].count do
+              if frame[:tokens][:a].count > 0 or frame[:tokens][:r].count > 0
+                builder.tokens do
                   frame[:tokens][:a].each do |token_id|
-                    builder.token id: token_id
+                    builder.token(flags: 'a', idref: token_id)
                   end
-                end
-              end
 
-              if frame[:tokens][:r].count > 0
-                builder.tokens flags: 'r', n: frame[:tokens][:r].count do
                   frame[:tokens][:r].each do |token_id|
-                    builder.token id: token_id
+                    builder.token(flags: 'r', idref: token_id)
                   end
                 end
               end
@@ -146,7 +182,7 @@ module PROIEL::Dictionary
     end
 
     def index_homographs!
-      @lemmata.keys.group_by { |l| l.split(',').first }.each do |m, homographs|
+      @lemmata.keys.group_by { |l| l.split(/[,#]/).first }.each do |m, homographs|
         if homographs.count > 1
           homographs.each do |form|
             @lemmata[form][:homographs] = homographs.reject { |homograph| homograph == form }
@@ -157,20 +193,9 @@ module PROIEL::Dictionary
 
     def index_token!(token)
       if token.lemma and token.part_of_speech
-        encoded_lemma = [token.lemma, token.part_of_speech].join(',')
-
-        @lemmata[encoded_lemma] ||= {
-          lemma: token.lemma,
-          part_of_speech: token.part_of_speech,
-          distribution: {},
-          glosses: {},
-          homographs: [],
-          paradigm: {},
-          n: 0,
-          valency: {},
-        }
+        lemma = initialize_lemma!(token.lemma, token.part_of_speech)
 
-        lemma = @lemmata[encoded_lemma]
+        lemma[:n] += 1
 
         lemma[:distribution][token.source.id] ||= 0
         lemma[:distribution][token.source.id] += 1
@@ -179,7 +204,6 @@ module PROIEL::Dictionary
         lemma[:paradigm][token.morphology][token.form] ||= 0
         lemma[:paradigm][token.morphology][token.form] += 1
 
-        lemma[:n] += 1
 
         # Find verbal nodes
         if token.part_of_speech[/^V/]