proiel 1.1.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 39313c422eb3b2d2f3ad565c0cde3cbd3ddb5271
-  data.tar.gz: 1edadad95bbaad82d4d7ab1a9cc409f8e80d3a74
+SHA256:
+  metadata.gz: b0df361b20a949a5a7c51f1055470507b8b152af79737a896762ac269ac62c20
+  data.tar.gz: 2346771429bd177c4233e470c8e1830871001b2a511e311ef4cb70520f19687f
 SHA512:
-  metadata.gz: 41bf1b5bcb3c8d8318128ea146b2609d02942d711553876d71c29cafc948312e79e8cd2e448fef751ca25c685c3f0d57a924004a46bdb3496a8f9913772e3e48
-  data.tar.gz: add1511098c62bdd4ee59fdd53e55b4b331595a5a5e02320e97dadff194e8b0b96fb24bd48511a02933d70a83208e0f5d40093a49eec959917177bf59589cbb7
+  metadata.gz: 4c2195b08451ee0208aec4f80d7c6edfba1b0ecd2c0405d8a797a5e9fc8a8b135c6a9997d7a1b7b5a52519a3d8275e2be1d0b9389bd698b371372180254202a9
+  data.tar.gz: eb23ab51a1e7607dd4453e84820aa558636e5ebef7c7caf96824cb0f6dd17af7edaa34ad03efc663d1e66b60b3d63ea2d8053726c86b6435efe3a27f50ea53ba

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2015 Marius L. Jøhndal
+Copyright (c) 2015-2016 Marius L. Jøhndal
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -12,7 +12,7 @@ PROIEL annotation scheme and the PROIEL XML-based interchange format.
 
 ## Installation
 
-To install this library you need Ruby 2.1 or newer.
+This library requires Ruby >= 2.4. Install as
 
 ```shell
 gem install proiel
@@ -35,7 +35,7 @@ bundle
 ```
 
 To download a sample treebank, initialize a new git repository and add the
-[PROIEL treebank](http://proiel.github.io) as a submodule:
+[PROIEL treebank](https://proiel.github.io) as a submodule:
 
 ```shell
 git init

data/lib/proiel.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2015-2016 Marius L. Jøhndal
+# Copyright (c) 2015-2018 Marius L. Jøhndal
 #
 # See LICENSE in the top-level source directory for licensing terms.
 #++
@@ -10,6 +10,12 @@ require 'ostruct'
 require 'sax-machine'
 require 'memoist'
 require 'nokogiri'
+require 'singleton'
+require 'erb'
+require 'open3'
+require 'set'
+require 'builder'
+require 'csv'
 
 require 'proiel/version'
 require 'proiel/utils'
@@ -27,3 +33,12 @@ require 'proiel/source'
 require 'proiel/div'
 require 'proiel/sentence'
 require 'proiel/token'
+require 'proiel/dictionary'
+require 'proiel/dictionary/builder'
+require 'proiel/lemma'
+require 'proiel/visualization'
+require 'proiel/chronology'
+require 'proiel/valency'
+require 'proiel/dictionary/builder'
+require 'proiel/alignment'
+require 'proiel/language'

data/lib/proiel/alignment.rb

@@ -0,0 +1,3 @@
+module PROIEL::Alignment; end
+
+require 'proiel/alignment/builder'

data/lib/proiel/alignment/builder.rb

@@ -0,0 +1,220 @@
+module PROIEL
+  module Alignment
+    module Builder
+      # This computes a matrix of original and translation sentences that are
+      # aligned. For now, this function does not handle translation sentences that
+      # are unaligned (this is tricky to handle robustly!). As the current treebank
+      # collection stands this is an issue that *should* not arise so this is for
+      # now a reasonable approximation.
+      def self.compute_matrix(alignment, source, blacklist = [], log_directory = nil)
+        matrix1 = group_backwards(alignment, source, blacklist)
+        raise unless matrix1.map { |r| r[:original] }.flatten.compact == alignment.sentences.map(&:id)
+
+        matrix2 = group_forwards(alignment, source, blacklist)
+        raise unless matrix2.map { |r| r[:translation] }.flatten.compact == source.sentences.map(&:id)
+
+        if log_directory
+          # Verify that both texts are still in the correct sequence
+          File.open(File.join(log_directory, "#{source.id}1"), 'w') do |f|
+            matrix1.map do |x|
+              f.puts x.inspect
+            end
+          end
+
+          File.open(File.join(log_directory, "#{source.id}2"), 'w') do |f|
+            matrix2.map do |x|
+              f.puts x.inspect
+            end
+          end
+        end
+
+        matrix = []
+        iter1 = { i: 0, m: matrix1 }
+        iter2 = { i: 0, m: matrix2 }
+
+        loop do
+          # Take from matrix1 unless we have a translation
+          while iter1[:i] < iter1[:m].length and iter1[:m][iter1[:i]][:translation].empty?
+            matrix << iter1[:m][iter1[:i]]
+            iter1[:i] += 1
+          end
+
+          # Take from matrix2 unless we have an original
+          while iter2[:i] < iter2[:m].length and iter2[:m][iter2[:i]][:original].empty?
+            matrix << iter2[:m][iter2[:i]]
+            iter2[:i] += 1
+          end
+
+          if iter1[:i] < iter1[:m].length and iter2[:i] < iter2[:m].length
+            # Now the two should match provided alignments are sorted the same way,
+            # so take one from each. If they don't match outright, we may have a case
+            # of swapped sentence orders or a gap (one sentence unaligned in one of
+            # the texts surrounded by two sentences that are aligned to the same
+            # sentence in the other text). We'll try to repair this by merging bits
+            # from the next row in various combinations.
+            #
+            # When adding to the new mateix, pick original from matrix1 and
+            # translation from matrix2 so that the original textual order is
+            # preserved
+            if repair(matrix, iter1, 0, iter2, 0) or
+
+               repair(matrix, iter1, 1, iter2, 0) or
+               repair(matrix, iter1, 0, iter2, 1) or
+               repair(matrix, iter1, 1, iter2, 1) or
+
+               repair(matrix, iter1, 2, iter2, 0) or
+               repair(matrix, iter1, 0, iter2, 2) or
+               repair(matrix, iter1, 2, iter2, 1) or
+               repair(matrix, iter1, 1, iter2, 2) or
+               repair(matrix, iter1, 2, iter2, 2) or
+
+               repair(matrix, iter1, 3, iter2, 0) or
+               repair(matrix, iter1, 0, iter2, 3) or
+               repair(matrix, iter1, 3, iter2, 1) or
+               repair(matrix, iter1, 1, iter2, 3) or
+               repair(matrix, iter1, 3, iter2, 2) or
+               repair(matrix, iter1, 2, iter2, 3) or
+               repair(matrix, iter1, 3, iter2, 3) or
+
+               repair(matrix, iter1, 4, iter2, 0) or
+               repair(matrix, iter1, 0, iter2, 4) or
+               repair(matrix, iter1, 4, iter2, 1) or
+               repair(matrix, iter1, 1, iter2, 4) or
+               repair(matrix, iter1, 4, iter2, 2) or
+               repair(matrix, iter1, 2, iter2, 4) or
+               repair(matrix, iter1, 4, iter2, 3) or
+               repair(matrix, iter1, 3, iter2, 4) or
+               repair(matrix, iter1, 4, iter2, 4)
+            else
+              STDERR.puts iter1[:i], iter1[:m][iter1[:i]].inspect
+              STDERR.puts iter2[:i], iter2[:m][iter2[:i]].inspect
+              raise
+            end
+          else
+            raise unless iter1[:i] == iter1[:m].length and iter2[:i] == iter2[:m].length
+            break
+          end
+        end
+
+        if log_directory
+          File.open(File.join(log_directory, "#{source.id}3"), 'w') do |f|
+            matrix.map do |x|
+              f.puts x.inspect
+            end
+          end
+        end
+
+        raise unless matrix.map { |r| r[:original]    }.flatten.compact == alignment.sentences.map(&:id)
+        raise unless matrix.map { |r| r[:translation] }.flatten.compact == source.sentences.map(&:id)
+
+        matrix
+      end
+
+      private
+
+      def self.group_forwards(alignment, source, blacklist = [])
+        # Make an original to translation ID mapping
+        mapping = {}
+
+        source.sentences.each do |sentence|
+          mapping[sentence.id] = []
+
+          next if blacklist.include?(sentence.id)
+
+          mapping[sentence.id] = sentence.inferred_alignment(alignment).map(&:id)
+        end
+
+        # Translate to a pairs of ID arrays, chunk original IDs that share at least
+        # one translation ID, then reduce the result so we get an array of m-to-n
+        # relations
+        mapping.map do |v, k|
+          { original: k, translation: [v] }
+        end.chunk_while do |x, y|
+          !(x[:original] & y[:original]).empty?
+        end.map do |chunk|
+          chunk.inject do |a, v|
+            a[:original] += v[:original]
+            a[:translation] += v[:translation]
+            a
+          end
+        end.map do |row|
+          { original: row[:original].uniq, translation: row[:translation] }
+        end
+      end
+
+      def self.group_backwards(alignment, source, blacklist = [])
+        # Make an original to translation ID mapping
+        mapping = {}
+
+        alignment.sentences.each do |sentence|
+          mapping[sentence.id] = []
+        end
+
+        source.sentences.each do |sentence|
+          next if blacklist.include?(sentence.id)
+
+          original_ids = sentence.inferred_alignment(alignment).map(&:id)
+
+          original_ids.each do |original_id|
+            mapping[original_id] << sentence.id
+          end
+        end
+
+        # Translate to a pairs of ID arrays, chunk original IDs that share at least
+        # one translation ID, then reduce the result so we get an array of m-to-n
+        # relations
+        mapping.map do |k, v|
+          { original: [k], translation: v }
+        end.chunk_while do |x, y|
+          !(x[:translation] & y[:translation]).empty?
+        end.map do |chunk|
+          chunk.inject do |a, v|
+            a[:original] += v[:original]
+            a[:translation] += v[:translation]
+            a
+          end
+        end.map do |row|
+          { original: row[:original], translation: row[:translation].uniq }
+        end
+      end
+
+      def self.repair_merge_cells(iter, delta, field)
+        matrix, i = iter[:m], iter[:i]
+        (0..delta).map { |j| matrix[i + j][field] }.inject(&:+)
+      end
+
+      def self.select_unaligned(iter, delta, field, check_field)
+        matrix, i = iter[:m], iter[:i]
+        (0..delta).select { |j| matrix[i + j][check_field].empty? }.map { |j| matrix[i + j][field] }.flatten
+      end
+
+      def self.repair(matrix, iter1, delta1, iter2, delta2)
+        o1 = repair_merge_cells(iter1, delta1, :original)
+        o2 = repair_merge_cells(iter2, delta2, :original)
+
+        t1 = repair_merge_cells(iter1, delta1, :translation)
+        t2 = repair_merge_cells(iter2, delta2, :translation)
+
+        u1 = select_unaligned(iter1, delta1, :original, :translation)
+        u2 = select_unaligned(iter2, delta2, :translation, :original)
+
+        if o1.sort - u1 == o2.sort.uniq and t1.sort.uniq == t2.sort - u2
+          unless delta1.zero? and delta2.zero?
+            STDERR.puts "Assuming #{delta1 + 1}/#{delta2 + 1} swapped sentence order:"
+            STDERR.puts ' * ' + (0..delta1).map { |j| iter1[:m][iter1[:i] + j].inspect }.join(' + ')
+            STDERR.puts ' * ' + (0..delta2).map { |j| iter2[:m][iter2[:i] + j].inspect }.join(' + ')
+          end
+
+          matrix << { original: o1, translation: t2 }
+
+          iter1[:i] += delta1 + 1
+          iter2[:i] += delta2 + 1
+
+          true
+        else
+          false
+        end
+      end
+    end
+  end
+end

data/lib/proiel/annotation_schema.rb

@@ -22,10 +22,17 @@ module PROIEL
 
     # Creates a new annotation schema object.
     def initialize(xml_object)
-      @part_of_speech_tags = make_part_of_speech_tags(xml_object).freeze
-      @relation_tags = make_relation_tags(xml_object).freeze
-      @morphology_tags = make_morphology_tags(xml_object).freeze
-      @information_status_tags = make_information_status_tags(xml_object).freeze
+      if xml_object
+        @part_of_speech_tags = make_part_of_speech_tags(xml_object).freeze
+        @relation_tags = make_relation_tags(xml_object).freeze
+        @morphology_tags = make_morphology_tags(xml_object).freeze
+        @information_status_tags = make_information_status_tags(xml_object).freeze
+      else
+        @part_of_speech_tags = {}.freeze
+        @relation_tags = {}.freeze
+        @morphology_tags = {}.freeze
+        @information_status_tags = {}.freeze
+      end
     end
 
     # @return [Hash<String,RelationTagDefinition>] definition of primary relation tags

data/lib/proiel/chronology.rb

@@ -0,0 +1,80 @@
+#--
+# Copyright (c) 2016-2017 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+
+# Methods for parsing chronological descriptions.  Extra care is taken to get
+# the interpretation of centuries and ranges involving the transition between 1
+# BC and AD 1 correct.
+module PROIEL::Chronology
+  # Computes the chronological midpoint of a chronological description.
+  #
+  # @param s [String] chronological description
+  #
+  # @return [Integer]
+  #
+  # @example
+  #   midpoint('1000')         # => 1000
+  #   midpoint('1000 BC')      # => -1000
+  #   midpoint('1000-1020')    # => 1010
+  def self.midpoint(s)
+    i = parse(s)
+
+    if i.is_a?(Array)
+      # Handle missing Julian year 0 by shifting years after 1 BC down by 1 and then shifting the midpoint back
+      # up again unless negative
+      if i.first < 0 and i.last > 0
+        y = (i.first + i.last - 1) / 2.0
+        if y < 0
+          y.floor
+        else
+          (y + 1).floor
+        end
+      else
+        ((i.first + i.last) / 2.0).floor # a non-integer midpoint is within the year of the integer part
+      end
+    elsif i.is_a?(Integer)
+      i
+    else
+      raise ArgumentError, 'integer or array expected'
+    end
+  end
+
+  # Parses a chronological description. The syntax of chronological
+  # descriptions is explained in the [PROIEL XML
+  # documentation](http://proiel.github.io/handbook/developer/proielxml.html#chronological-data).
+  #
+  # @param s [String] chronological description
+  #
+  # @return [Integer, Array<Integer,Integer>]
+  #
+  # @example
+  #   parse('1000')         # => 1000
+  #   parse('1000 BC')      # => -1000
+  #   parse('1000-1020')    # => [1000,1020]
+  #   parse('1000 BC-1020') # => [-1000,1020]
+  def self.parse(s)
+    case s
+    when /^\s*(?:c\.\s+)?(\d+)(\s+BC)?\s*$/
+      i = $1.to_i
+      multiplier = $2 ? -1 : 1
+      (i * multiplier).to_i.tap do |i|
+        # There is no year zero in the Julian calendar
+        raise ArgumentError, 'invalid year' if i.zero?
+      end
+    when /^\s*(1st|2nd|3rd|\d+th)\s+c\.\s*$/
+      a = $1.to_i * 100
+      [a - 99, a]
+    when /^\s*(1st|2nd|3rd|\d+th)\s+c\.\s+BC\s*$/
+      a = -$1.to_i * 100
+      [a, a + 99]
+    when /^\s*(?:c\.\s+)?\d+(\s+BC)?\s*-\s*(c\.\s+)?\d+(\s+BC)?\s*$/
+      s.split('-').map { |i| parse(i) }.tap do |from, to|
+        raise ArgumentError, 'invalid range' unless from < to
+      end
+    else
+      raise ArgumentError, 'unexpected format'
+    end
+  end
+end

data/lib/proiel/dictionary.rb

@@ -0,0 +1,79 @@
+#--
+# Copyright (c) 2018 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+module PROIEL
+  class Dictionary < TreebankObject
+    # @return [Treebank] treebank that this source belongs to
+    attr_reader :treebank
+
+    # @return [String] language of the source as an ISO 639-3 language tag
+    attr_reader :language
+
+    # @return [String] dialect of the source
+    attr_reader :dialect
+
+    # @return [DateTime] export time for the dictionary
+    attr_reader :export_time
+
+    # @return [Hash] all lemmata in the dictionary
+    attr_reader :lemmata
+
+    # @return [Integer] number of lemmata in the dictionary
+    attr_reader :n
+
+    # @return [Hash] all sources in the dictionary
+    attr_reader :sources
+
+    # Creates a new dictionary object.
+    def initialize(parent, export_time, language, dialect, xml = nil)
+      @treebank = parent
+
+      raise ArgumentError, 'string or nil expected' unless export_time.nil? or export_time.is_a?(String)
+      @export_time = export_time.nil? ? nil : DateTime.parse(export_time).freeze
+
+      @language = language.freeze
+      @dialect = dialect ? dialect.freeze : nil
+
+      @lemmata = {}
+      @sources = {}
+      @n = 0
+
+      from_xml(xml) if xml
+    end
+
+    # FIXME
+    def id
+      @language
+    end
+
+    private
+
+    def from_xml(xml)
+      xml.sources.each do |s|
+        @sources[s.idref] = { license: nullify(s.license), n: nullify(s.n, :int) }
+      end
+
+      xml.lemmata.each do |l|
+        @lemmata[l.lemma] ||= {}
+        @lemmata[l.lemma][l.part_of_speech] = Lemma.new(self, l)
+        @n += 1
+      end
+    end
+
+    def nullify(s, type = nil)
+      case s
+      when NilClass, /^\s*$/
+        nil
+      else
+        case type
+        when :int
+          s.to_i
+        else
+          s.to_s
+        end
+      end
+    end
+  end
+end