proiel 1.0.0

data/lib/proiel/proiel_xml/reader.rb

@@ -0,0 +1,237 @@
+#--
+# Copyright (c) 2015 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+module PROIEL
+  module PROIELXML
+    # @api private
+    module Reader
+      # Parsing class for `slash` elements.
+      class Slash
+        include SAXMachine
+
+        attribute :'target-id', as: :target_id, class: Integer, required: true
+        attribute :relation, required: true
+      end
+
+      # Parsing class for `token` elements.
+      class Token
+        include SAXMachine
+
+        attribute :id, class: Integer, required: true
+        attribute :'head-id', as: :head_id, class: Integer
+        attribute :form
+        attribute :lemma
+        attribute :'part-of-speech', as: :part_of_speech
+        attribute :morphology
+        attribute :relation
+        attribute :'empty-token-sort', as: :empty_token_sort
+        attribute :'citation-part', as: :citation_part
+        attribute :'presentation-before', as: :presentation_before
+        attribute :'presentation-after', as: :presentation_after
+        attribute :'antecedent-id', as: :antecedent_id, class: Integer
+        attribute :'information-status', as: :information_status
+        attribute :'contrast-group', as: :contrast_group
+        attribute :'foreign-ids', as: :foreign_ids
+
+        elements :slash, as: :slashes, class: Slash
+      end
+
+      # Parsing class for `sentence` elements.
+      class Sentence
+        include SAXMachine
+
+        attribute :id, class: Integer, required: true
+        attribute :status, class: Symbol, default: :unannotated
+        attribute :'presentation-before', as: :presentation_before
+        attribute :'presentation-after', as: :presentation_after
+
+        elements :token, as: :tokens, class: Token
+      end
+
+      # Parsing class for `div` elements.
+      class Div
+        include SAXMachine
+
+        attribute :id
+        attribute :'presentation-before', as: :presentation_before
+        attribute :'presentation-after', as: :presentation_after
+
+        element :title
+        elements :sentence, as: :sentences, class: Sentence
+      end
+
+      # Parsing class for `source` elements.
+      class Source
+        include SAXMachine
+
+        attribute :id, required: true
+        attribute :language, required: true
+
+        element :title
+        element :author
+        element :citation_part
+        element :principal
+        element :funder
+        element :distributor
+        element :distributor_address
+        element :date
+        element :license
+        element :license_url
+        element :reference_system
+        element :editor
+        element :editorial_note
+        element :annotator
+        element :reviewer
+        element :electronic_text_editor
+        element :electronic_text_title
+        element :electronic_text_version
+        element :electronic_text_publisher
+        element :electronic_text_place
+        element :electronic_text_date
+        element :electronic_text_original_url
+        element :electronic_text_license
+        element :electronic_text_license_url
+        element :printed_text_editor
+        element :printed_text_title
+        element :printed_text_edition
+        element :printed_text_publisher
+        element :printed_text_place
+        element :printed_text_date
+        elements :div, as: :divs, class: Div
+      end
+
+      # Parsing class for `relations/value` elements.
+      class RelationValue
+        include SAXMachine
+
+        attribute :tag, required: true
+        attribute :summary, required: true
+        attribute :primary, required: true
+        attribute :secondary, required: true
+      end
+
+      # Parsing class for `relations` elements.
+      class Relations
+        include SAXMachine
+
+        elements :value, as: :values, class: RelationValue
+      end
+
+      # Parsing class for `parts_of_speech/value` elements.
+      class PartOfSpeechValue
+        include SAXMachine
+
+        attribute :tag, required: true
+        attribute :summary, required: true
+      end
+
+      # Parsing class for `parts_of_speech` elements.
+      class PartsOfSpeech
+        include SAXMachine
+
+        elements :value, as: :values, class: PartOfSpeechValue
+      end
+
+      # Parsing class for `morphology/field/value` elements.
+      class MorphologyValue
+        include SAXMachine
+
+        attribute :tag, required: true
+        attribute :summary, required: true
+      end
+
+      # Parsing class for `morphology/field` elements.
+      class MorphologyField
+        include SAXMachine
+
+        attribute :tag, required: true
+
+        elements :value, as: :values, class: MorphologyValue
+      end
+
+      # Parsing class for `morphology` elements.
+      class Morphology
+        include SAXMachine
+
+        elements :field, as: :fields, class: MorphologyField
+      end
+
+      # Parsing class for `information_statuses/value` elements.
+      class InformationStatusValue
+        include SAXMachine
+
+        attribute :tag, required: true
+        attribute :summary, required: true
+      end
+
+      # Parsing class for `information_statuses` elements.
+      class InformationStatuses
+        include SAXMachine
+
+        elements :value, as: :values, class: InformationStatusValue
+      end
+
+      # Parsing class for `annotation` elements.
+      class Annotation
+        include SAXMachine
+
+        element :relations, class: Relations
+        element :parts_of_speech, as: :parts_of_speech, class: PartsOfSpeech
+        element :morphology, class: Morphology
+        element :information_statuses, as: :information_statuses, class: InformationStatuses
+      end
+
+      # Parsing class for `proiel` elements.
+      class Proiel
+        include SAXMachine
+
+        attribute :'export-time', as: :export_time
+        attribute :'schema-version', as: :schema_version, required: true
+
+        elements :source, as: :sources, class: Source
+        element :annotation, class: Annotation
+      end
+
+      # Top-level parsing class for a PROIEL XML file.
+      class TreebankFile
+        include SAXMachine
+
+        element :proiel, class: Proiel
+      end
+
+      # Parses PROIEL XML data.
+      #
+      # This does not automatically validate the PROIEL XML. If given an
+      # invalid PROIEL XML file, parsing is likely to succeed but the returned
+      # objects will be in an inconsistent state.
+      #
+      # @see parse_io
+      #
+      # @param xml [String] PROIEL XML to parse
+      #
+      # @return [TreebankFile]
+      #
+      def self.parse_xml(xml)
+        TreebankFile.parse(xml)
+      end
+
+      # Parses a PROIEL XML file.
+      #
+      # This does not automatically validate the PROIEL XML. If given an
+      # invalid PROIEL XML file, parsing is likely to succeed but the returned
+      # objects will be in an inconsistent state.
+      #
+      # @see parse_xml
+      #
+      # @param io [IO] stream representing the PROIEL XML file
+      #
+      # @return [TreebankFile]
+      #
+      def self.parse_io(io)
+        parse_xml(io.read)
+      end
+    end
+  end
+end

data/lib/proiel/proiel_xml/schema.rb

@@ -0,0 +1,81 @@
+#--
+# Copyright (c) 2015 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+module PROIEL
+  module PROIELXML
+    # Functionality concerned with PROIEL XML schema loading and versioning.
+    # Functionality for validation using a PROIEL XML schema is found in
+    # {PROIEL::PROIELXML::Validator}.
+    #
+    # @api private
+    module Schema
+      # Returns the current version of the PROIEL XML schema.
+      #
+      # @return [String] schema version number
+      #
+      def self.current_proiel_xml_schema_version
+        '2.0'
+      end
+
+      # Invalid PROIEL XML schema version error.
+      #
+      # This represents an error that occurs when an unknown PROIEL XML schema
+      # version number is encountered or one that could not be parsed.
+      class InvalidSchemaVersion < RuntimeError; end
+
+      # Opens a PROIEL XML schema file and peek at the schema version number
+      # that the file claims it conforms to.
+      #
+      # @return [String] schema version number
+      #
+      # @raise InvalidSchemaVersion
+      #
+      def self.check_schema_version_of_xml_file(filename)
+        doc = Nokogiri::XML(File.read(filename))
+
+        if doc and doc.root and doc.root.name == 'proiel'
+          case doc.root.attr('schema-version')
+          when '2.0'
+            '2.0'
+          when NilClass
+            '1.0'
+          else
+            raise InvalidSchemaVersion, 'invalid schema version number'
+          end
+        else
+          raise InvalidSchemaVersion, 'top-level XML element not found'
+        end
+      end
+
+      # Loads a PROIEL XML schema.
+      #
+      # @return [Nokogiri::XML::Schema] schema version number
+      #
+      # @raise RuntimeError
+      #
+      def self.load_proiel_xml_schema(schema_version)
+        filename = proiel_xml_schema_filename(schema_version)
+
+        Nokogiri::XML::Schema(File.open(filename).read)
+      end
+
+      # Determines the filename of a specific version of the PROIEL XML schema.
+      #
+      # @return [String] filename
+      #
+      # @raise ArgumentError
+      #
+      def self.proiel_xml_schema_filename(schema_version)
+        if schema_version == '1.0' or schema_version == '2.0'
+          File.join(File.dirname(__FILE__),
+                    "proiel-#{schema_version}",
+                    "proiel-#{schema_version}.xsd")
+        else
+          raise ArgumentError, 'invalid schema version'
+        end
+      end
+    end
+  end
+end

data/lib/proiel/proiel_xml/validator.rb

@@ -0,0 +1,177 @@
+#--
+# Copyright (c) 2015 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+module PROIEL
+  module PROIELXML
+    # A validator object that uses an XML schema as well as additional
+    # integrity checks to validate a PROIEL XML file. Functionality for
+    # loading the XML schema and checking the PROIEL XML version number is
+    # found in {PROIEL::PROIELXML::Schema}.
+    class Validator
+      # Returns an array of error messages generated during validation.
+      attr_reader :errors
+
+      # Creates a new validator for a PROIEL XML file.
+      #
+      # @param filename [String] name of PROIEL XML file to validate
+      #
+      def initialize(filename)
+        @filename = filename
+        @errors = []
+      end
+
+      # Checks if the PROIEL XML file is valid. This checks for
+      # well-formedness, a valid schema version, validation against the schema
+      # and referential integrity.
+      #
+      # If invalid, `errors` will contain error messages.
+      #
+      # @return [true, false]
+      #
+      def valid?
+        wellformed? and valid_schema_version? and validates? and has_referential_integrity?
+      end
+
+      # Checks if the PROIEL XML file is well-formed XML.
+      #
+      # If not well-formed, an error message will be appended to `errors`.
+      #
+      # @return [true, false]
+      #
+      def wellformed?
+        begin
+          Nokogiri::XML(File.read(@filename)) { |config| config.strict }
+
+          true
+        rescue Nokogiri::XML::SyntaxError => _
+          @errors << 'XML file is not wellformed'
+
+          false
+        end
+      end
+
+      # Checks if the PROIEL XML file has a valid schema version number.
+      #
+      # If invalid, an error message will be appended to `errors`.
+      #
+      # @return [true, false]
+      #
+      def valid_schema_version?
+        schema_version = PROIEL::PROIELXML::Schema.check_schema_version_of_xml_file(@filename)
+
+        if schema_version.nil?
+          @errors << 'invalid schema version'
+
+          false
+        else
+          true
+        end
+      rescue PROIEL::PROIELXML::Schema::InvalidSchemaVersion => e
+        @errors << e.message
+
+        false
+      end
+
+      # Checks if the PROIEL XML file validates against the schema.
+      #
+      # If invalid, error messages will be appended to `errors`.
+      #
+      # @return [true, false]
+      #
+      def validates?
+        doc = Nokogiri::XML(File.read(@filename))
+
+        schema_version = PROIEL::PROIELXML::Schema.check_schema_version_of_xml_file(@filename)
+
+        schema = PROIEL::PROIELXML::Schema.load_proiel_xml_schema(schema_version)
+        r = schema.validate(doc)
+
+        if r.empty?
+          true
+        else
+          @errors += r.map { |e| "Line #{e.line}: #{e.message}" }
+
+          false
+        end
+      end
+
+      # Checks the referential integrity of the PROIEL XML file.
+      #
+      # If inconsistencies are found, error messages will be appended to `errors`.
+      #
+      # @return [true, false]
+      #
+      def has_referential_integrity?
+        tb = PROIEL::Treebank.new
+        tb.load_from_xml(@filename)
+
+        errors = []
+
+        # Pass 1: keep track of all object IDs and look for duplicates
+        sentence_ids = {}
+        token_ids = {}
+
+        tb.sources.each do |source|
+          source.divs.each do |div|
+            div.sentences.each do |sentence|
+              errors << "Repeated sentence ID #{sentence.id}" if sentence_ids.key?(sentence.id)
+              sentence_ids[sentence.id] = true
+
+              sentence.tokens.each do |token|
+                errors << "Repeated token ID #{token.id}" if token_ids.key?(token.id)
+                token_ids[token.id] = { sentence: sentence.id, div: div.id, source: source.id }
+              end
+            end
+          end
+        end
+
+        # Pass 2: check object ID references
+        tb.sources.each do |source|
+          source.tokens.each do |token|
+            # Head IDs and slash IDs should be sentence internal
+            check_reference_locality(errors, token, token_ids, :head_id, token.head_id, domain: :sentence, allow_nil: true)
+
+            token.slashes.each do |_, target_id|
+              check_reference_locality(errors, token, token_ids, :slash_id, target_id, domain: :sentence, allow_nil: false)
+            end
+
+            # Antecedent IDs should be source internal
+            check_reference_locality(errors, token, token_ids, :antecedent_id, token.antecedent_id, domain: :source, allow_nil: true)
+          end
+        end
+
+        # Pass 3: verify that all features are defined
+        # TBD
+
+        if errors.empty?
+          true
+        else
+          @errors += errors
+
+          false
+        end
+      end
+
+      private
+
+      def check_reference_locality(errors, token, token_ids, attribute_name,
+                                   attribute_value, domain: :sentence, allow_nil: false)
+        if attribute_value
+          referenced_token = token_ids[attribute_value]
+
+          if referenced_token.nil?
+            errors << "Token #{token.id}: #{attribute_name} references an unknown token"
+          elsif referenced_token[domain] != token.send(domain).id
+            errors << "Token #{token.id}: #{attribute_name} references a token in a different #{domain}"
+          end
+        elsif allow_nil
+          # Everything is fine...
+        else
+          errors << "Token #{token.id}: #{attribute_name} is null"
+        end
+      end
+    end
+  end
+end

data/lib/proiel/sentence.rb

@@ -0,0 +1,191 @@
+#--
+# Copyright (c) 2015 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+module PROIEL
+  # A sentence object in a treebank.
+  class Sentence < TreebankObject
+    extend Memoist
+
+    # @return [Fixnum] ID of the sentence
+    attr_reader :id
+
+    # @return [Div] parent div object
+    attr_reader :div
+
+    # @return [Symbol] annotation status of sentence
+    attr_reader :status
+
+    # @return [nil, String] presentation material before sentence
+    attr_reader :presentation_before
+
+    # @return [nil, String] presentation material after sentence
+    attr_reader :presentation_after
+
+    # Creates a new sentence object.
+    def initialize(parent, id, status, presentation_before, presentation_after, &block)
+      @div = parent
+
+      raise ArgumentError, 'integer expected' unless id.is_a?(Integer)
+      @id = id
+
+      raise ArgumentError, 'string or symbol expected' unless status.is_a?(String) or status.is_a?(Symbol)
+      @status = status.to_sym
+
+      raise ArgumentError, 'string or nil expected' unless presentation_before.nil? or presentation_before.is_a?(String)
+      @presentation_before = presentation_before.freeze
+
+      raise ArgumentError, 'string or nil expected' unless presentation_after.nil? or presentation_after.is_a?(String)
+      @presentation_after = presentation_after.freeze
+
+      @children = block.call(self) if block_given?
+    end
+
+    # @return [Source] parent source object
+    def source
+      @div.source
+    end
+
+    # @return [Treebank] parent treebank object
+    def treebank
+      @div.source.treebank
+    end
+
+    # @return [String] language of the sentence as an ISO 639-3 language tag
+    def language
+      source.language
+    end
+
+    memoize :language
+
+    # @return [String] the complete citation for the sentence
+    def citation
+      [source.citation_part, citation_part].join(' ')
+    end
+
+    # Computes an appropriate citation component for the sentence.
+    #
+    # The computed citation component must be concatenated with the citation
+    # component provided by the source to produce a complete citation.
+    #
+    # @see citation
+    #
+    # @return [String] the citation component
+    def citation_part
+      tc = @children.select(&:has_citation?)
+      x = tc.first ? tc.first.citation_part : nil
+      y = tc.last ? tc.last.citation_part : nil
+
+      Citations.citation_make_range(x, y)
+    end
+
+    # Returns the printable form of the sentence with all token forms and any
+    # presentation data.
+    #
+    # @return [String] the printable form of the sentence
+    def printable_form(options = {})
+      [presentation_before,
+       @children.map { |t| t.printable_form(options) },
+       presentation_after].compact.join
+    end
+
+    # Checks if the sentence is reviewed.
+    #
+    # A sentence has been reviewed if its `status` is `:reviewed`.
+    #
+    # @return [true,false]
+    def reviewed?
+      @status == :reviewed
+    end
+
+    # Checks if the sentence is annotated.
+    #
+    # Since only annotated sentences can be reviewed, a sentence is annotated
+    # if its `status` is either `:reviewed` or `:annotated`.
+    #
+    # @return [true,false]
+    def annotated?
+      @status == :reviewed or @status == :annotated
+    end
+
+    # Checks if the sentence is unannotated.
+    #
+    # A sentence is unannotated if its `status` is `:unannotated`.
+    #
+    # @return [true,false]
+    def unannotated?
+      @status == :unannotated
+    end
+
+    # Builds a syntax graph for the dependency annotation of the sentence and
+    # inserts a dummy root node. The graph is represented as a hash of
+    # hashes.  Each hash contains the ID of the token, its relation (to its
+    # syntatically dominating token) and a list of secondary edges.
+    #
+    # @return [Hash] a single graph with a dummy root node represented as a hash
+    #
+    # @example
+    #
+    #   sentence.syntax_graph # => [id: nil, relation: nil, children: [{ id: 1000, relation: "pred", children: [ { id: 1001, relation: "xcomp", children: [], slashes: [["xsub", 1000]]}]}], slashes: []]
+    #
+    def syntax_graph
+      { id: nil, relation: nil, children: syntax_graphs, slashes: [] }
+    end
+
+    # Builds syntax graphs for the dependency annotation of the sentence.
+    # Multiple graphs may be returned as the function does not insert an
+    # empty dummy root node. Each graph is represented as a hash of hashes.
+    # Each hash contains the ID of the token, its relation (to its
+    # syntatically dominating token) and a list of secondary edges.
+    #
+    # @return [Array] zero or more syntax graphs represented as hashes
+    #
+    # @example Get a single syntax graph with a dummy root node
+    #
+    #  sentence.syntax_graphs # => [{ id: 1000, relation: "pred", children: [ { id: 1001, relation: "xcomp", children: [], slashes: [["xsub", 1000]]}]}]
+    #
+    def syntax_graphs
+      Array.new.tap do |graphs|
+        token_map = {}
+
+        # Pass 1: create new attribute hashes for each token and index each hash by token ID
+        @children.each do |token|
+          token_map[token.id] =
+            {
+              id: token.id,
+              relation: token.relation,
+              children: [],
+              slashes: token.slashes,
+            }
+        end
+
+        # Pass 2: append attribute hashes for tokens with a head ID to the head's children list; append attribute hashes for tokens without a head ID to the list of graphs to return
+        @children.each do |token|
+          if token.head_id
+            token_map[token.head_id][:children] << token_map[token.id]
+          else
+            graphs << token_map[token.id]
+          end
+        end
+      end
+    end
+
+    # Finds all tokens in the sentence.
+    #
+    # @return [Enumerator] tokens in the sentence
+    #
+    # @example Iterating tokens
+    #   tokens.each { |t| puts t.id }
+    #
+    # @example Create an array with only empty tokens
+    #   tokens.select(&:is_empty?)
+    #
+    # @example Counting tokens
+    #   puts tokens.count #=> 200
+    #
+    def tokens
+      @children.to_enum
+    end
+  end
+end