proiel-cli 0.1.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bfb4db333ae791d4171490c7b3d316364b205729
-  data.tar.gz: fd001265bdcc8e75e49fc1e2a188f7c63471bb62
+  metadata.gz: cf5056d98706541003c897f25b470145a26dab43
+  data.tar.gz: 3bc9d72f373deec98554ec4bab9d0482f427a6b6
 SHA512:
-  metadata.gz: 29cb26e28f22486db097b982d7e2cd4783f2b38d8d6c13fb06309fb25377791cd8248c88627b7d5b72c3adfe0e958c9696b2508fcb975e9a8fdf516a9825963b
-  data.tar.gz: b565292467456b10415039f0464bdfafcae5fecf55747dd3b9ec2473313ba08789d49431ecb06d649596bfd98031640481efccd9b5be58b6b565d13dcc3a1161
+  metadata.gz: 6f4a512e9c4f35ccfdd84c4ec1c7d2bed4c5089862802c6171247817b64bb00ba9a3dadf2e82b7ba12bee2b647b86d0faf705e402e34ba6f74c0bdc3a8547796
+  data.tar.gz: c14b7972c896f8f47dbf764b69796c011c9827b715fbf324b2bcca58f82d30acad7f0906d084e3e654299bb5cc22a77547c7fa6a7ef0e6d1c135227cf69701d9

data/bin/proiel

@@ -6,7 +6,7 @@ $:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib })
 require 'proiel/cli'
 
 Mercenary.program(:proiel) do |p|
-  p.version PROIEL::VERSION
+  p.version PROIEL::CLI::VERSION
   p.description 'proiel is a command-line interface for PROIEL treebanks'
   p.syntax 'proiel <subcommand> [options]'
 

data/lib/proiel/cli/commands/convert.rb

@@ -16,6 +16,9 @@ module PROIEL
               f.option 'remove-syntax', '--remove-syntax', 'Remove syntactic annotation (relation, head ID and slashes)'
               f.option 'remove-information-structure', '--remove-information-structure', 'Remove informtion structure annotation (antecedent ID, information status and contrast group)'
               f.option 'remove-status', '--remove-status', 'Remove sentence status (i.e. revert all sentences to unannotated status)'
+              f.option 'remove-alignments', '--remove-alignments', 'Remove alignments'
+              f.option 'remove-annotator', '--remove-annotator', 'Remove annotator information'
+              f.option 'remove-reviewer', '--remove-reviewer', 'Remove reviewer information'
               f.option 'remove-empty-divs', '--remove-empty-divs', 'Remove div elements that do not contain any sentences'
               f.action { |args, options| process(args, options, PROIEL::Converter::PROIELXML) }
             end

data/lib/proiel/cli/commands/tokenize.rb

@@ -48,8 +48,9 @@ module PROIEL
           citation_part = nil
 
           body.each_with_index do |sd_body, i|
-            builder.div(title: sd_body[:title]) do
-              sd_body[:contents].split(/(@[^ ]+|§[^ ]+)/).map do |s|
+            builder.div do
+              builder.title sd_body[:title]
+              sd_body[:contents].split(/(@[^ ]+|§[^ ]+ )/).map do |s|
                 if s[0] == '§' or s[0] == '@'
                   s
                 else
@@ -57,16 +58,16 @@ module PROIEL
                   # sentence-breaking punctuation like periods and question marks, but
                   # after the punctuation mark and characters typically used in pairs,
                   # like brackets and apostrophes.
-                  s.gsub(/([\.:;\?!]+[\s†\]\)"']*)/, '\1|')
+                  s.gsub(/([\.:;\?!]+[\s†\]\)"']*|\s*[\n\r]+)/, '\1|')
                 end
               end.join.split('|').each_with_index do |s_body, j|
-                builder.sentence(status_tag: 'unannotated') do
+                builder.sentence(status: 'unannotated') do
                   leftover_before = ''
 
                   # Preserve linebreaks in the text.
-                  s_body.gsub!(/\s*[\n\r]/, "\u2028")
+                  s_body.gsub!(/\s*[\n\r]+/, "\u2028")
 
-                  s_body.scan(/([^@§\p{Word}]*)([\p{Word}]+|@[^ ]+|§[^ ]+)([^@§\p{Word}]*)/).each do |(before, form, after)|
+                  s_body.scan(/([^@§\p{Word}]*)([\p{Word}]+|@[^ ]+|§[^ ]+ )([^@§\p{Word}]*)/).each do |(before, form, after)|
                     case form
                     when /^@(.*)$/
                       leftover_before += before unless before.nil?
@@ -74,15 +75,15 @@ module PROIEL
                       leftover_before += after unless after.nil?
                     when /^§(.*)$/
                       leftover_before += before unless before.nil?
-                      citation_part = $1
+                      citation_part = $1.strip
                       leftover_before += after unless after.nil?
                     else
                       before = leftover_before + before
                       leftover_before = ''
 
-                      attrs = { citation_part: citation_part, form: form }
-                      attrs[:presentation_before] = before unless before == ''
-                      attrs[:presentation_after] = after unless after == ''
+                      attrs = { :"citation-part" => citation_part, form: form }
+                      attrs[:"presentation-before"] = before unless before == ''
+                      attrs[:"presentation-after"] = after unless after == ''
 
                       builder.token(attrs)
                     end

data/lib/proiel/cli/commands/words.rb

@@ -0,0 +1,36 @@
+module PROIEL
+  module Commands
+    class Words < Command
+      class << self
+        def init_with_program(prog)
+          prog.command(:words) do |c|
+            c.syntax 'words [options] filename(s)'
+            c.description 'Extract a word list'
+
+            c.action do |args, options|
+              if args.empty?
+                STDERR.puts 'Missing filename(s). Use --help for more information.'
+              else
+                process(args, options)
+              end
+            end
+          end
+        end
+
+        def process(args, options)
+          tb = PROIEL::Treebank.new
+
+          args.each do |filename|
+            STDERR.puts "Reading #{filename}...".green if options['verbose']
+
+            tb.load_from_xml(filename)
+          end
+
+          tb.sources.map { |s| s.tokens.map(&:form) }.flatten.sort.uniq.each do |form|
+            STDOUT.puts form
+          end
+        end
+      end
+    end
+  end
+end

data/lib/proiel/cli/converters/conll-u.rb

@@ -57,6 +57,7 @@ module PROIEL
                       t.relation,
                       t.empty_token_sort,
                       t.slashes.map { |relation, target_id| [id_to_number[target_id], relation] },
+                      t.citation_part,
                       self
                      )
           end
@@ -164,8 +165,9 @@ module PROIEL
         attr_reader :language
         attr_reader :empty_token_sort
         attr_reader :form
+        attr_reader :citation_part
 
-        def initialize(id, head_id, form, lemma, part_of_speech, language, morphology, relation, empty_token_sort, slashes, sentence)
+        def initialize(id, head_id, form, lemma, part_of_speech, language, morphology, relation, empty_token_sort, slashes, citation_part, sentence)
           @id = id
           @head_id = head_id
           @form = form
@@ -178,6 +180,7 @@ module PROIEL
           @slashes = slashes
           @sentence = sentence
           @features = (morphology ? map_morphology(morphology) : '' )
+          @citation_part = "ref=" + (citation_part ? citation_part : "").gsub(/\s/, '_')
           @upos = nil
         end
 
@@ -351,7 +354,7 @@ module PROIEL
            @head_id, 
            (@head_id == 0 ? 'root' : @relation), # override non-root relations on root until we've found out how to handle unembedded reports etc
            '_', # slashes here
-           '_'].join("\t")
+           @citation_part].join("\t")
         end
 
         def to_s

data/lib/proiel/cli/converters/proielxml.rb

@@ -5,7 +5,7 @@ module PROIEL
         def process(tb, options)
           builder = Builder::XmlMarkup.new(target: STDOUT, indent: 2)
           builder.instruct! :xml, version: '1.0', encoding: 'UTF-8'
-          builder.proiel('export-time' => DateTime.now.xmlschema, 'schema-version' => '2.0') do
+          builder.proiel('export-time' => DateTime.now.xmlschema, 'schema-version' => '2.1') do
             builder.annotation do
               builder.relations do
                 tb.annotation_schema.relation_tags.each do |tag, value|
@@ -45,14 +45,24 @@ module PROIEL
             end
 
             tb.sources.each do |source|
-              builder.source(id: source.id, language: source.language) do
+              mandatory_features = %i(id language)
+              optional_features = []
+              optional_features += %i(alignment_id) unless options['remove-alignments']
+
+              builder.source(grab_features(source, mandatory_features, optional_features)) do
                 PROIEL::Treebank::METADATA_ELEMENTS.each do |field|
                   builder.tag!(field.to_s.gsub('_', '-'), source.send(field)) if source.send(field)
                 end
 
                 source.divs.each do |div|
                   if include_div?(div, options)
-                    builder.div(grab_features(div, %i(), %i(presentation_before presentation_after))) do
+                    mandatory_features = %i()
+
+                    optional_features = []
+                    optional_features += %i(presentation_before presentation_after)
+                    optional_features += %i(alignment_id) unless options['remove-alignments']
+
+                    builder.div(grab_features(div, mandatory_features, optional_features)) do
                       builder.title div.title if div.title
 
                       div.sentences.each do |sentence|
@@ -62,6 +72,11 @@ module PROIEL
                           optional_features = [] # we do it this way to preserve the order of status and presentation_* so that diffing files is easier
                           optional_features += %i(status) unless options['remove-status']
                           optional_features += %i(presentation_before presentation_after)
+                          optional_features += %i(alignment_id) unless options['remove-alignments']
+                          optional_features += %i(annotated_at) unless options['remove-annotator']
+                          optional_features += %i(reviewed_at) unless options['remove-reviewer']
+                          optional_features += %i(annotated_by) unless options['remove-annotator']
+                          optional_features += %i(reviewed_by) unless options['remove-reviewer']
 
                           builder.sentence(grab_features(sentence, mandatory_features, optional_features)) do
                             sentence.tokens.each do |token|
@@ -83,6 +98,8 @@ module PROIEL
                                 mandatory_features << :empty_token_sort
                               end
 
+                              optional_features += %i(alignment_id) unless options['remove-alignments']
+
                               attrs = grab_features(token, mandatory_features, optional_features)
 
                               unless token.slashes.empty? or options['remove-syntax'] # this extra test avoids <token></token> style XML

data/lib/proiel/cli/version.rb

@@ -1,5 +1,5 @@
 module PROIEL
   module CLI
-    VERSION = '0.1.1'
+    VERSION = '1.0.0'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: proiel-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Marius L. Jøhndal
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-06 00:00:00.000000000 Z
+date: 2016-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: builder
@@ -59,14 +59,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -204,6 +204,7 @@ files:
 - lib/proiel/cli/commands/info.rb
 - lib/proiel/cli/commands/tokenize.rb
 - lib/proiel/cli/commands/validate.rb
+- lib/proiel/cli/commands/words.rb
 - lib/proiel/cli/converters/conll-u.rb
 - lib/proiel/cli/converters/conll-u/morphology.rb
 - lib/proiel/cli/converters/conll-u/syntax.rb
@@ -235,7 +236,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: A command-line interface for working with PROIEL treebanks