proiel 1.1.0 → 1.3.1

data/lib/proiel/proiel_xml/schema.rb

@@ -16,7 +16,7 @@ module PROIEL
       # @return [String] schema version number
       #
       def self.current_proiel_xml_schema_version
-        '2.1'
+        '3.0'
       end
 
       # Invalid PROIEL XML schema version error.
@@ -41,6 +41,8 @@ module PROIEL
             '2.0'
           when '2.1'
             '2.1'
+          when '3.0'
+            '3.0'
           when NilClass
             '1.0'
           else
@@ -70,7 +72,7 @@ module PROIEL
       # @raise ArgumentError
       #
       def self.proiel_xml_schema_filename(schema_version)
-        if schema_version == '1.0' or schema_version == '2.0' or schema_version == '2.1'
+        if schema_version == '1.0' or schema_version == '2.0' or schema_version == '2.1' or schema_version == '3.0'
           File.join(File.dirname(__FILE__),
                     "proiel-#{schema_version}",
                     "proiel-#{schema_version}.xsd")

data/lib/proiel/proiel_xml/validator.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2015 Marius L. Jøhndal
+# Copyright (c) 2015-2017 Marius L. Jøhndal
 #
 # See LICENSE in the top-level source directory for licensing terms.
 #++
@@ -16,9 +16,11 @@ module PROIEL
       # Creates a new validator for a PROIEL XML file.
       #
       # @param filename [String] name of PROIEL XML file to validate
+      # @param aligned_filename [NilClass, String] name of PROIEL XML file to validate alignments against
       #
-      def initialize(filename)
+      def initialize(filename, aligned_filename = nil)
         @filename = filename
+        @aligned_filename = aligned_filename
         @errors = []
       end
 
@@ -41,15 +43,13 @@ module PROIEL
       # @return [true, false]
       #
       def wellformed?
-        begin
-          Nokogiri::XML(File.read(@filename)) { |config| config.strict }
+        Nokogiri::XML(File.read(@filename)) { |config| config.strict }
 
-          true
-        rescue Nokogiri::XML::SyntaxError => _
-          @errors << 'XML file is not wellformed'
+        true
+      rescue Nokogiri::XML::SyntaxError => _
+        @errors << 'XML file is not wellformed'
 
-          false
-        end
+        false
       end
 
       # Checks if the PROIEL XML file has a valid schema version number.
@@ -154,6 +154,27 @@ module PROIEL
           end
         end
 
+        # Pass 5: if div is aligned, sentences and tokens within should belong
+        # to aligned div(s); if sentence aligned, tokens within should belong
+        # to aligned sentence(s). Skip if no alignment_id on source (see pass
+        # 4) or if aligned source not available.
+        if @aligned_filename
+          aligned_tb = PROIEL::Treebank.new
+          aligned_tb.load_from_xml(@aligned_filename)
+
+          tb.sources.each do |source|
+            if source.alignment_id
+              aligned_source = aligned_tb.find_source(source.alignment_id)
+
+              if aligned_source
+                check_alignment_integrity(errors, source, aligned_source)
+              else
+                errors << "Aligned source not available in treebank"
+              end
+            end
+          end
+        end
+
         # Decide if there were any errors
         if errors.empty?
           true
@@ -182,6 +203,52 @@ module PROIEL
           errors << "Token #{token.id}: #{attribute_name} is null"
         end
       end
+
+      def check_alignment_integrity(errors, source, aligned_source)
+        source.divs.each do |div|
+          target_sentences =
+            div.sentences.map do |sentence|
+              target_tokens =
+                sentence.tokens.select(&:alignment_id).map do |token|
+                  # Check that target token exists in aligned source
+                  aligned_token = aligned_source.treebank.find_token(token.alignment_id)
+
+                  if aligned_token
+                    aligned_token
+                  else
+                    errors << "Token #{token.id}: aligned to token #{aligned_source.id}:#{token.alignment_id} which does not exist"
+                    nil
+                  end
+                end
+
+              inferred_target_sentences = target_tokens.compact.map(&:sentence).sort_by(&:id).uniq
+
+              if sentence.alignment_id
+                a = sentence.alignment_id.to_s.split(',').sort.join(',')
+                i = inferred_target_sentences.map(&:id).sort.join(',')
+
+                # FIXME: handle i.empty? case, in which we have to use a and check that the objects exist
+                if a != i
+                  errors << "Sentence #{sentence.id}: aligned to sentence #{aligned_source.id}:#{a} but inferred alignment is #{aligned_source.id}:#{i}"
+                end
+              end
+
+              inferred_target_sentences
+            end
+
+          inferred_target_divs = target_sentences.flatten.compact.map(&:div).uniq
+
+          if div.alignment_id
+            a = div.alignment_id.to_s.split(',').sort.join(',')
+            i = inferred_target_divs.map(&:id).sort.join(',')
+
+            # FIXME: handle i.empty? case, in which we have to use a and check that the objects exist
+            if a != i
+              errors << "Div #{div.id}: aligned to div #{aligned_source.id}:#{a} but inferred alignment is #{aligned_source.id}:#{i}"
+            end
+          end
+        end
+      end
     end
   end
 end

data/lib/proiel/sentence.rb

@@ -57,10 +57,14 @@ module PROIEL
       raise ArgumentError, 'integer or nil expected' unless alignment_id.nil? or alignment_id.is_a?(Integer)
       @alignment_id = alignment_id
 
-      raise ArgumentError, 'XML schema date time or nil expected' unless annotated_at.nil? or PROIEL::Utilities.xmlschema_datetime?(annotated_at)
+      unless annotated_at.nil? or PROIEL::Utilities.xmlschema_datetime?(annotated_at)
+        raise ArgumentError, 'XML schema date time or nil expected'
+      end
       @annotated_at = annotated_at ? DateTime.xmlschema(annotated_at).freeze : nil
 
-      raise ArgumentError, 'XML schema date time or nil expected' unless reviewed_at.nil? or PROIEL::Utilities.xmlschema_datetime?(reviewed_at)
+      unless reviewed_at.nil? or PROIEL::Utilities.xmlschema_datetime?(reviewed_at)
+        raise ArgumentError, 'XML schema date time or nil expected'
+      end
       @reviewed_at = reviewed_at ? DateTime.xmlschema(reviewed_at).freeze : nil
 
       raise ArgumentError, 'string or nil expected' unless annotated_by.nil? or annotated_by.is_a?(String)
@@ -113,10 +117,13 @@ module PROIEL
     # Returns the printable form of the sentence with all token forms and any
     # presentation data.
     #
+    # @param custom_token_formatter [Lambda] formatting function for tokens
+    # which is passed the token as its sole argument
+    #
     # @return [String] the printable form of the sentence
-    def printable_form(options = {})
+    def printable_form(custom_token_formatter: nil)
       [presentation_before,
-       @children.map { |t| t.printable_form(options) },
+       @children.reject(&:is_empty?).map { |t| t.printable_form(custom_token_formatter: custom_token_formatter) },
        presentation_after].compact.join
     end
 
@@ -217,5 +224,21 @@ module PROIEL
     def tokens
       @children.to_enum
     end
+
+    # Returns the aligned sentence if any.
+    #
+    # @return [Sentence, NilClass] aligned sentence
+    def alignment(aligned_source)
+      alignment_id ? aligned_source.treebank.find_sentence(alignment_id) : nil
+    end
+
+    # Returns inferred aligned sentences if any.
+    #
+    # @return [Array<Sentence>] inferred aligned sentences
+    def inferred_alignment(aligned_source)
+      tokens.select(&:alignment_id).map do |token|
+        token.alignment(aligned_source)
+      end.flatten.compact.map(&:sentence).uniq
+    end
   end
 end

data/lib/proiel/source.rb

@@ -15,6 +15,9 @@ module PROIEL
     # @return [String] language of the source as an ISO 639-3 language tag
     attr_reader :language
 
+    # @return [String] dialect of the source
+    attr_reader :dialect
+
     # @return [DateTime] export time for the source
     attr_reader :export_time
 
@@ -26,11 +29,15 @@ module PROIEL
     attr_reader :alignment_id
 
     # Creates a new source object.
-    def initialize(parent, id, export_time, language, metadata, alignment_id, &block)
+    def initialize(parent, id, export_time, language, dialect, metadata, alignment_id, &block)
       @treebank = parent
       @id = id.freeze
-      @export_time = DateTime.parse(export_time).freeze
+
+      raise ArgumentError, 'string or nil expected' unless export_time.nil? or export_time.is_a?(String)
+      @export_time = export_time.nil? ? nil : DateTime.parse(export_time).freeze
+
       @language = language.freeze
+      @dialect = dialect ? dialect.freeze : nil
       @metadata = metadata.freeze
 
       raise ArgumentError, 'string or nil expected' unless alignment_id.nil? or alignment_id.is_a?(String)
@@ -47,9 +54,12 @@ module PROIEL
     # Returns the printable form of the source with all token forms and any
     # presentation data.
     #
+    # @param custom_token_formatter [Lambda] formatting function for tokens
+    # which is passed the token as its sole argument
+    #
     # @return [String] the printable form of the source
-    def printable_form(options = {})
-      @children.map { |d| d.printable_form(options) }.compact.join
+    def printable_form(custom_token_formatter: nil)
+      @children.map { |d| d.printable_form(custom_token_formatter: custom_token_formatter) }.compact.join
     end
 
     # Accesses metadata fields.

data/lib/proiel/statistics.rb

@@ -26,11 +26,11 @@ module PROIEL
 
       x_mean = x.reduce(&:+).to_f / x.size
       y_mean = y.reduce(&:+).to_f / y.size
-      x_sqsum = x.reduce(0.0) { |sum, n| sum + n ** 2 }
+      x_sqsum = x.reduce(0.0) { |sum, n| sum + n**2 }
       xy_sum = x.zip(y).reduce(0.0) { |sum, (m, n)| sum + m * n }
 
       sxy = xy_sum - x.length * x_mean * y_mean
-      sx2 = x_sqsum - x.length * (x_mean ** 2)
+      sx2 = x_sqsum - x.length * (x_mean**2)
 
       beta = sxy / sx2
       alfa = y_mean - beta * x_mean

data/lib/proiel/token.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2015-2016 Marius L. Jøhndal
+# Copyright (c) 2015-2017 Marius L. Jøhndal
 #
 # See LICENSE in the top-level source directory for licensing terms.
 #++
@@ -160,12 +160,13 @@ module PROIEL
     # Returns the printable form of the token with any presentation data.
     #
     # @param custom_token_formatter [Lambda] formatting function for tokens
+    # which is passed the token as its sole argument
     #
     # @return [String] the printable form of the token
     def printable_form(custom_token_formatter: nil)
       printable_form =
         if custom_token_formatter
-          custom_token_formatter.call(id, form)
+          custom_token_formatter.call(self)
         else
           form
         end
@@ -281,7 +282,7 @@ module PROIEL
     #
     # @return [Array<Token>] descendents
     def descendents
-      dependents.map { |dependent| [dependent ] + dependent.descendents }.flatten
+      dependents.map { |dependent| [dependent] + dependent.descendents }.flatten
     end
 
     memoize :descendents
@@ -393,22 +394,29 @@ module PROIEL
       common_ancestors(other_token, inclusive: inclusive).first
     end
 
+    # Returns the aligned token if any.
+    #
+    # @return [Token, NilClass] aligned token
+    def alignment(aligned_source)
+      alignment_id ? aligned_source.treebank.find_token(alignment_id) : nil
+    end
+
     private
 
     # FIXME: extract this from the header of the PROIEL XML file instead and
     # subclass PositionalTag
-    POS_POSITIONAL_TAG_SEQUENCE = %i(major minor)
+    POS_POSITIONAL_TAG_SEQUENCE = %i(major minor).freeze
 
     # FIXME: extract this from the header of the PROIEL XML file instead and
     # subclass PositionalTag
     MORPHOLOGY_POSITIONAL_TAG_SEQUENCE = %i(
       person number tense mood voice gender case degree strength inflection
-    )
+    ).freeze
 
     NULL_PARTS_OF_SPEECH = {
       'V' => 'V-',
       'C' => 'C-',
       'P' => 'Pp',
-    }
+    }.freeze
   end
 end

data/lib/proiel/tokenization.rb

@@ -24,7 +24,7 @@ module PROIEL
 
       patterns = JSON.parse(File.read(filename))
 
-      regexes = patterns.map { |l, p| [l, self.make_regex(p)] }.to_h
+      regexes = patterns.map { |l, p| [l, make_regex(p)] }.to_h
 
       @@regexes ||= {}
       @@regexes.merge!(regexes)
@@ -59,6 +59,8 @@ module PROIEL
       form and form.length > 1
     end
 
+    WORD_PATTERN = /([^[\u{E000}-\u{F8FF}][[:word:]]]+)/.freeze
+
     # Splits a token form using the tokenization patterns that apply for a
     # the specified language. Tokenization patterns must already have been
     # loaded.
@@ -73,9 +75,9 @@ module PROIEL
       raise ArgumentError, 'invalid language tag' unless language_tag.is_a?(String)
       raise ArgumentError, 'invalid form' unless form.is_a?(String)
 
-      if form[/[^[:word:]]+/]
+      if form[WORD_PATTERN]
         # Split on any non-word character like a space or punctuation
-        form.split(/([^[:word:]]+)/)
+        form.split(WORD_PATTERN)
       elsif @@regexes.key?(language_tag) and form[@@regexes[language_tag]]
         # Apply language-specific pattern
         form.match(@@regexes[language_tag]).captures

data/lib/proiel/treebank.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2015 Marius L. Jøhndal
+# Copyright (c) 2015-2018 Marius L. Jøhndal
 #
 # See LICENSE in the top-level source directory for licensing terms.
 #++
@@ -23,9 +23,13 @@ module PROIEL
     # @return [Array<Source>] sources in the treebank
     attr_reader :sources
 
+    # @return [Array<Dictionary>] dictionaries in the treebank
+    attr_reader :dictionaries
+
     # Available metadata elements for sources.
     METADATA_ELEMENTS = %i(
       title
+      alternative_title
       author
       citation_part
       principal
@@ -55,13 +59,16 @@ module PROIEL
       printed_text_publisher
       printed_text_place
       printed_text_date
-    )
+      chronology_composition
+      chronology_manuscript
+    ).freeze
 
     # Creates a new treebank object.
     def initialize
       @annotation_schema = nil
       @schema_version = nil
       @sources = []
+      @dictionaries = []
 
       @source_index = {}
       @div_index = {}
@@ -85,12 +92,18 @@ module PROIEL
         tf = PROIELXML::Reader.parse_io(f)
 
         tf.proiel.sources.each do |s|
-          @sources << Source.new(self, s.id, tf.proiel.export_time, s.language,
+          @sources << Source.new(self, s.id, tf.proiel.export_time, s.language, s.dialect,
                                  bundle_metadata(s), s.alignment_id) do |source|
             build_divs(s, source)
           end
 
-          index_objects!(@sources.last)
+          index_source_objects!(@sources.last)
+        end
+
+        tf.proiel.dictionaries.each do |s|
+          @dictionaries << Dictionary.new(self, tf.proiel.export_time, s.language, s.dialect, s)
+
+          index_dictionary_objects!(@dictionaries.last)
         end
 
         annotation_schema = AnnotationSchema.new(tf.proiel.annotation)
@@ -103,7 +116,7 @@ module PROIEL
           # FIXME: consolidate export times? This is a design flaw in PROIEL XML
           # 2.0: export time ought to be per source not per PROIEL XML file, so
           # not clear what to do here. Pass it down to the source object?
-          #@export_time = tf.proiel.export_time
+          # @export_time = tf.proiel.export_time
         else
           raise SchemaMismatch
         end
@@ -198,7 +211,7 @@ module PROIEL
       end
     end
 
-    def index_objects!(source)
+    def index_source_objects!(source)
       @source_index[source.id] = source
 
       source.divs.each do |div|
@@ -213,5 +226,9 @@ module PROIEL
         end
       end
     end
+
+    def index_dictionary_objects!(dictionary)
+      # TODO
+    end
   end
 end

data/lib/proiel/utils.rb

@@ -13,4 +13,3 @@ module PROIEL
     end
   end
 end
-

data/lib/proiel/valency.rb

@@ -0,0 +1,5 @@
+module PROIEL::Valency; end
+
+require 'proiel/valency/obliqueness'
+require 'proiel/valency/arguments'
+require 'proiel/valency/lexicon'