proiel 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6271d2f5d29934447660cf7a2ba1f416a1171b50
+  data.tar.gz: 467914536f5f6794fa84729227b4f0ddff62c2dd
+SHA512:
+  metadata.gz: 4ec4b448baff57c7faf9b31861667e18cc3f9297dc946c31d6e9e3b491050b39af8e0894eea5ca82c2051fb45d2d90c16b5046568ce780a79f7027beaff323e3
+  data.tar.gz: 1a8c3cf8c2c29b11904bcc67e4e4a0aa557c38bf8c0dbf66deaef36a404cb7deceae2fbefd7fc21d82c208408cf515e291ee2dc960bef56820108cfe390d7847

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2015 Marius L. Jøhndal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,99 @@
+# PROIEL treebank utility library
+
+This is a utility library for reading and manipulating treebanks that use the
+PROIEL annotation scheme and the PROIEL XML-based interchange format.
+
+## Installation
+
+To install this library you need Ruby 2.1 or newer.
+
+```shell
+gem install proiel
+```
+
+## Getting started
+
+The recommended way to use this library in your application is with `bundler`.
+Create a `Gemfile` with the following content:
+
+```ruby
+source 'https://rubygems.org'
+gem 'proiel', '~> 1.0'
+```
+
+and then execute
+
+```shell
+bundle
+```
+
+To download a sample treebank, initialize a new git repository and add the
+[PROIEL treebank](http://proiel.github.io) as a submodule:
+
+```shell
+git init
+mkdir vendor
+git submodule add --depth 1 https://github.com/proiel/proiel-treebank.git vendor/proiel-treebank
+```
+
+Here is a skeleton programme to get you started. Save this as `myproject.rb`:
+
+```ruby
+#!/usr/bin/env ruby
+require 'proiel'
+
+tb = PROIEL::Treebank.new
+Dir[File.join('vendor', 'proiel-treebank', '*.xml')].each do |filename|
+  puts "Reading #{filename}..."
+  tb.load_from_xml(filename)
+end
+
+tb.sources.each do |source|
+  source.divs.each do |div|
+    div.sentences.each do |sentence|
+      sentence.tokens.each do |token|
+        # Do something
+      end
+    end
+  end
+end
+```
+
+You can now run this as:
+
+```shell
+bundle exec ruby myproject.rb
+```
+
+See the [wiki](https://github.com/proiel/proiel/wiki) for more information.
+
+## Versioning
+
+`proiel` aims to adhere to [Semantic Versioning 2.0.0](http://semver.org/spec/v2.0.0.html). This means that a patch version or minor version should not break backward compatibility of a public API, and that breaking changes should only be introduced with new major versions. When specifying a dependency on this gem it is best practice to use a pessimistic version constraint with two digits of precision:
+
+```ruby
+spec.add_dependency 'proiel', '~> 1.0'
+```
+
+## Development
+
+Check out the git repository from GitHub and run `bin/setup` to install
+all development dependencies. Then run `rake` to run the tests.
+
+You can also run `bin/console` for an interactive prompt to experiment with.
+
+To install a development version of this gem, run `bundle exec rake install`.
+
+To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the gem to [rubygems.org](https://rubygems.org).
+
+## Documentation
+
+Documentation can be generated using YARD:
+
+```sh
+yard
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/proiel/proiel.

data/bin/console

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+require 'bundler/setup'
+require 'proiel'
+
+require 'pry'
+Pry.start

data/bin/setup

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install

data/lib/proiel/annotation_schema.rb

@@ -0,0 +1,127 @@
+#--
+# Copyright (c) 2015 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+module PROIEL
+  # A representation of the annotation schema found in the header of a PROIEL
+  # XML file. This should not be confused with the PROIEL XML schema, which is
+  # used for validating the XML in a PROIEL XML file.
+  class AnnotationSchema
+    # @return [Hash<String,PartOfSpeechTagDefinition>] definition of part of speech tags
+    attr_reader :part_of_speech_tags
+
+    # @return [Hash<String,RelationTagDefinition>] definition of relation tags
+    attr_reader :relation_tags
+
+    # @return [Hash<Symbol,Hash<String,MorphologyFieldTagDefinition>>] definition of morphology tags
+    attr_reader :morphology_tags
+
+    # @return [Hash<String,InformationStatusTagDefinition>] definition of information status tags
+    attr_reader :information_status_tags
+
+    # Creates a new annotation schema object.
+    def initialize(xml_object)
+      @part_of_speech_tags = make_part_of_speech_tags(xml_object).freeze
+      @relation_tags = make_relation_tags(xml_object).freeze
+      @morphology_tags = make_morphology_tags(xml_object).freeze
+      @information_status_tags = make_information_status_tags(xml_object).freeze
+    end
+
+    # @return [Hash<String,RelationTagDefinition>] definition of primary relation tags
+    def primary_relations
+      @relation_tags.select { |_, features| features.primary }
+    end
+
+    # @return [Hash<String,RelationTagDefinition>] definition of secondary relation tags
+    def secondary_relations
+      @relation_tags.select { |_, features| features.secondary }
+    end
+
+    # Tests for equality of two annotation schema objects.
+    #
+    # @return [true,false]
+    #
+    def ==(o)
+      @part_of_speech_tags.sort_by(&:first) == o.part_of_speech_tags.sort_by(&:first) and
+        @relation_tags.sort_by(&:first) == o.relation_tags.sort_by(&:first)
+    end
+
+    private
+
+    def make_tag_hash(element)
+      element.values.map { |e| [e.tag, yield(e)] }.compact.to_h
+    end
+
+    def make_relation_tags(xml_object)
+      make_tag_hash(xml_object.relations) do |e|
+        RelationTagDefinition.new(e.summary, e.primary == 'true', e.secondary == 'true')
+      end
+    end
+
+    def make_part_of_speech_tags(xml_object)
+      make_tag_hash(xml_object.parts_of_speech) do |e|
+        PartOfSpeechTagDefinition.new(e.summary)
+      end
+    end
+
+    def make_morphology_tags(xml_object)
+      xml_object.morphology.fields.map do |f|
+        v =
+          make_tag_hash(f) do |e|
+            MorphologyFieldTagDefinition.new(e.summary)
+          end
+        [f.tag, v]
+      end.to_h
+    end
+
+    def make_information_status_tags(xml_object)
+      make_tag_hash(xml_object.information_statuses) do |e|
+        InformationStatusTagDefinition.new(e.summary)
+      end
+    end
+  end
+
+  # A tag definitions.
+  #
+  # @abstract
+  class GenericTagDefinition
+    attr_reader :summary
+
+    def initialize(summary)
+      @summary = summary
+    end
+
+    # Tests equality of two tag definitions.
+    def ==(o)
+      @summary == o.summary
+    end
+  end
+
+  # Definition of an information status tag.
+  class InformationStatusTagDefinition < GenericTagDefinition; end
+
+  # Definition of a relation tag.
+  class RelationTagDefinition < GenericTagDefinition
+    attr_reader :primary
+    attr_reader :secondary
+
+    def initialize(summary, primary, secondary)
+      super(summary)
+
+      @primary = primary
+      @secondary = secondary
+    end
+
+    # Tests equality of two tag definitions.
+    def ==(o)
+      @summary == o.summary and @primary == o.primary and @secondary == o.secondary
+    end
+  end
+
+  # Definition of a morphology field tag.
+  class MorphologyFieldTagDefinition < GenericTagDefinition; end
+
+  # Definition of a part of speech tag.
+  class PartOfSpeechTagDefinition < GenericTagDefinition; end
+end

data/lib/proiel/citations.rb

@@ -0,0 +1,84 @@
+#--
+# Copyright (c) 2015 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+module PROIEL
+  module Citations
+    # Returns a citation range that spans `cit1` to `cit2`.
+    #
+    # The regular expression `dividers` is used to chunk the strings, and then
+    # the longest common prefix of chunks is removed from `cit2`. `dividers`
+    # should chosen so that the chunks match logical components of a citation,
+    # e.g. book titles, chapter numbers and section identifiers.
+    #
+    # @param cit1 [String] first citation in range
+    # @param cit2 [String] second citation in range
+    # @param dividers [Regexp] dividing elements between components of citation
+    #
+    # @return [String]
+    #
+    # @example
+    #   citation_make_range('Matt 5.16', 'Matt 5.27') # => "Matt 5.16–27"
+    #   citation_make_range('Matt 4.13', 'Matt 5.27') # => "Matt 4.13–5.27"
+    #
+    def self.citation_make_range(cit1, cit2, dividers: /([\s\.]+)/)
+      raise ArgumentError unless cit1.is_a?(String) or cit1.nil?
+      raise ArgumentError unless cit2.is_a?(String) or cit1.nil?
+
+      # Remove any nil and empty-string citation, and reduce a range that starts
+      # and ends with the same citation to a single citation.
+      c = [cit1, cit2].reject { |c| c.nil? || c.empty? }.uniq
+
+      case c.length
+      when 0
+        nil
+      when 1
+        c.first
+      else
+        s = citation_strip_prefix(cit1, cit2, dividers: dividers)
+        [cit1, s].reject(&:empty?).join("\u{2013}")
+      end
+    end
+
+    # Returns `cit2` without the longest prefix that `cit1` and `cit2` have in
+    # common.
+    #
+    # The longest common prefix is not computed from the raw strings `cit1` and
+    # `cit2` but from string chunks. The regular expression `dividers` is used
+    # to chunk the strings, and then the longest prefix of chunks is removed.
+    #
+    # `dividers` should chosen so that the chunks match logical componets of a
+    # citation, e.g. book titles, chapter numbers and section identifiers.
+    #
+    # @param cit1 [String] first citation in range
+    # @param cit2 [String] second citation in range
+    # @param dividers [Regexp] dividing elements between components of citation
+    #
+    # @return [String]
+    #
+    # @example
+    #   citation_strip_prefix('Matt 5.16', 'Matt 5.27') # => "27"
+    #   citation_strip_prefix('Matt 5.26', 'Matt 5.27') # => "27"
+    #   citation_strip_prefix('Matt 4.13', 'Matt 5.27') # => "5.27"
+    #
+    def self.citation_strip_prefix(cit1, cit2, dividers: /([\s\.]+)/u)
+      raise ArgumentError unless cit1.is_a?(String)
+      raise ArgumentError unless cit2.is_a?(String)
+
+      x, y = cit1.split(dividers), cit2.split(dividers)
+
+      # Interleave x and y but compensate for zip's behaviour when
+      # y.length < x.length
+      zipped = x.length >= y.length ? x.zip(y) : y.zip(x).map(&:reverse)
+
+      zipped.inject('') do |d, (a, b)|
+        if not d.empty? or a != b
+          d + (b || '')
+        else
+          ''
+        end
+      end
+    end
+  end
+end

data/lib/proiel/div.rb

@@ -0,0 +1,133 @@
+#--
+# Copyright (c) 2015 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+module PROIEL
+  # A div object in a treebank.
+  class Div < TreebankObject
+    extend Memoist
+
+    # Returns the ID of the div.
+    #
+    # PROIEL XML 2.0 lacks IDs for divs while later versions require them. For
+    # PROIEL XML 2.0 unique IDs are generated by PROIEL::Treebank.
+    #
+    # @return [Fixnum] ID of the div
+    attr_reader :id
+
+    # @return [Source] source that the div belongs to
+    attr_reader :source
+
+    # @return [nil, String] title of the div
+    attr_reader :title
+
+    # @return [nil, String] presentation material before form
+    attr_reader :presentation_before
+
+    # @return [nil, String] presentation material after form
+    attr_reader :presentation_after
+
+    # Creates a new div object.
+    def initialize(parent, id, title, presentation_before, presentation_after, &block)
+      @source = parent
+
+      raise ArgumentError, 'integer expected' unless id.is_a?(Integer)
+      @id = id
+
+      raise ArgumentError, 'string or nil expected' unless title.nil? or title.is_a?(String)
+      @title = title.freeze
+
+      raise ArgumentError, 'string or nil expected' unless presentation_before.nil? or presentation_before.is_a?(String)
+      @presentation_before = presentation_before.freeze
+
+      raise ArgumentError, 'string or nil expected' unless presentation_after.nil? or presentation_after.is_a?(String)
+      @presentation_after = presentation_after.freeze
+
+      @children = block.call(self) if block_given?
+    end
+
+    # @return [Treebank] parent treebank object
+    def treebank
+      @source.treebank
+    end
+
+    # @return [String] language of the div as an ISO 639-3 language tag
+    def language
+      source.language
+    end
+
+    memoize :language
+
+    # @return [String] a complete citation for the div
+    def citation
+      [source.citation_part, citation_part].join(' ')
+    end
+
+    # Computes an appropriate citation component for the div.
+    #
+    # The computed citation component must be concatenated with the citation
+    # component provided by the source to produce a complete citation.
+    #
+    # @see citation
+    #
+    # @return [String] the citation component
+    def citation_part
+      tc = tokens.select(&:has_citation?)
+      x = tc.first ? tc.first.citation_part : nil
+      y = tc.last ? tc.last.citation_part : nil
+
+      Citations.citation_make_range(x, y)
+    end
+
+    # Returns the printable form of the div with all token forms and any
+    # presentation data.
+    #
+    # @return [String] the printable form of the div
+    def printable_form(options = {})
+      [presentation_before,
+       @children.map { |s| s.printable_form(options) },
+       presentation_after].compact.join
+    end
+
+    # Finds all sentences in the div.
+    #
+    # @return [Enumerator] sentences in the div
+    #
+    # @example Iterating sentences
+    #   sentences.each { |s| puts s.id }
+    #
+    # @example Create an array with only reviewed sentences
+    #   sentences.select(&:reviewed?)
+    #
+    # @example Counting sentences
+    #   sentences.count #=> 200
+    #
+    def sentences
+      @children.to_enum
+    end
+
+    # Finds all tokens in the div.
+    #
+    # @return [Enumerator] tokens in the div
+    #
+    # @example Iterating tokens
+    #   tokens.each { |t| puts t.id }
+    #
+    # @example Create an array with only empty tokens
+    #   tokens.select(&:is_empty?)
+    #
+    # @example Counting tokens
+    #   puts tokens.count #=> 200
+    #
+    def tokens
+      Enumerator.new do |y|
+        @children.each do |sentence|
+          sentence.tokens.each do |token|
+            y << token
+          end
+        end
+      end
+    end
+  end
+end

data/lib/proiel/positional_tag.rb

@@ -0,0 +1,127 @@
+#--
+# Copyright (c) 2015 Marius L. Jøhndal
+#
+# See LICENSE in the top-level source directory for licensing terms.
+#++
+module PROIEL
+  # Represents a positional tag, which consists of one or more fields each with
+  # its own value. The default implementation is of a positional tag with no
+  # fields. The class should be subclassed and the `fields` method overridden
+  # to implement a non-empty positional tag.
+  #
+  # @abstract Subclass and override {#fields} to implement a custom positional tag class.
+  class PositionalTag
+    include Comparable
+
+    # Creates a new positional tag.
+    #
+    # @param value [String, Hash, PositionalTag] initial value
+    #
+    def initialize(value = nil)
+      @fields = Hash.new
+
+      case value
+      when NilClass
+      when String
+        set_value!(fields.zip(value.split('')).to_h)
+      when Hash
+        set_value!(value)
+      when PositionalTag
+        set_value!(value.to_h)
+      else
+        raise ArgumentError, 'expected nil, Hash, String or PositionalTag'
+      end
+    end
+
+    # Returns an integer, -1, 0 or 1, suitable for sorting the tag.
+    #
+    # @return [Integer]
+    #
+    def <=>(o)
+      to_s <=> o.to_s
+    end
+
+    # Returns the positional tag as a string.
+    #
+    # @return [String]
+    #
+    def to_s
+      # Iterate fields to ensure conversion of fields without a value to
+      # UNSET_FIELD.
+      fields.map { |field| self[field] }.join
+    end
+
+    # Checks if the tag is unitialized. The tag is uninitialized if no field
+    # has a value.
+    #
+    # @return [true, false]
+    #
+    def empty?
+      @fields.empty?
+    end
+
+    # Returns a hash representation of the tag. The keys are the names of each
+    # field as symbols, the values are the values of each field.
+    #
+    # @return [Hash<Symbol, String>]
+    #
+    def to_h
+      @fields
+    end
+
+    # Returns the value of a field. An field without a value is returned
+    # as `-`.
+    #
+    # @param field [String, Symbol] name of field
+    #
+    # @return [String]
+    #
+    def [](field)
+      field = field.to_sym
+
+      raise ArgumentError, "invalid field #{field}" unless fields.include?(field)
+
+      @fields[field] || UNSET_FIELD
+    end
+
+    # Assigns a value to a field. Removing any value from a field can be done
+    # by assigning `nil` or `-`.
+    #
+    # @param field [String, Symbol] name of field
+    # @param value [String, nil]
+    #
+    # @return [String]
+    #
+    def []=(field, value)
+      field = field.to_sym
+
+      raise ArgumentError, "invalid field #{field}" unless fields.include?(field)
+
+      if value == UNSET_FIELD or value.nil?
+        @fields.delete(field)
+      else
+        @fields.store(field, value)
+      end
+
+      value
+    end
+
+    # Returns the field names. This method should be overridden by
+    # implementations. The names should be returned as an array of symbols.
+    #
+    # @return [Array<Symbol>]
+    #
+    def fields
+      []
+    end
+
+    private
+
+    # The string representation of a field without a value.
+    UNSET_FIELD = '-'.freeze
+
+    def set_value!(o)
+      o.each { |k, v| self[k] = v }
+    end
+  end
+end