proiel-cli 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 98027bdd669bde3fa19db4f6159e3b36d5024f1f
-  data.tar.gz: 4e6019add4436629e2488fc03d19f3d040c48149
+SHA256:
+  metadata.gz: 7f03b2148dc4a7dff5176d490e8850507b4984df13dafd56b04dd81763708df7
+  data.tar.gz: 8e41729b4ab79c7f48c8b34f77ac65a58188e950879c473f41dcc1c74af17d4d
 SHA512:
-  metadata.gz: 301a294838fb2ee805bcabbdb5425d39815749c8d4ccea9e8f7883ccaee4fee09ffb0cd45ad8ea0bef06cfb7b80a241b4acb87d224079101863be2d8f0614c7a
-  data.tar.gz: ca1bbaadca8be10e714984ab8ae1f460e6c61d3e5c7b09882c8819e95aadcccc152c69e53e1bd0072be6fc30491c422a84c6886fbd4f7f203b17cfa517f92310
+  metadata.gz: 838988757f5ee2360496297516047fe2715da30c8327aa411b6d3f395284d02a33123c263c238874e596aedcb53bbe9aeb79d63363fd712cb3480357c54bf87a
+  data.tar.gz: 1f629b9b036776f23a0c80851ea80f69082e8f2df8675ed268e0afdc86ac2e94264fecbaedd686a087475cc29e2abb7d99b090a4179c48c62516c402ea5179b5

data/README.md

@@ -1,10 +1,17 @@
 # PROIEL command-line interface
 
+## Status
+
+[![Gem Version](https://badge.fury.io/rb/proiel-cli.svg)](http://badge.fury.io/rb/proiel-cli)
+[![Build Status](https://secure.travis-ci.org/proiel/proiel-cli.svg?branch=master)](http://travis-ci.org/proiel/proiel-cli?branch=master)
+
+## Description
+
 This is a command-line interface for manipulating PROIEL treebanks.
 
 ## Installation
 
-This library requires Ruby >= 2.1. Install as
+This library requires Ruby >= 2.4. Install as
 
 ```shell
 gem install proiel-cli
@@ -35,8 +42,16 @@ Bug reports and pull requests are welcome on [GitHub](https://github.com/proiel/
 
 ## Development
 
-To contribute to development, check out the git repository from [GitHub](https://github.com/proiel/proiel-cli) and run `bin/setup` to install all development dependencies. Then run `rake` to run the tests.
+To contribute to development, check out the git repository from [GitHub](https://github.com/proiel/proiel-cli) and run `bundle install` to install all development dependencies. Then run `rake` to run the tests.
 
 To install a development version of this gem, run `bundle exec rake install`.
 
-To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the gem to [rubygems.org](https://rubygems.org).
+### Releasing a new version
+
+To release a new version:
+
+1.  Update the version number in `lib/proiel/cli/version.rb`.
+2.  Run `bundle exec rake release`. This will:
+    *   Create a git tag for the version.
+    *   Push git commits and tags to the remote repository.
+    *   Push the `.gem` file to [rubygems.org](https://rubygems.org).

data/bin/proiel

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 STDOUT.sync = true
 
-$:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib })
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'proiel/cli'
 

data/lib/proiel/cli/commands/build.rb

@@ -0,0 +1,91 @@
+module PROIEL
+  module Commands
+    class Build < Command
+      class << self
+        def init_with_program(prog)
+          prog.command(:build) do |c|
+            c.syntax 'build resource'
+            c.description 'Build a derived resource'
+
+            c.command(:dictionary) do |f|
+              f.syntax 'output_filename [filename(s)]'
+              f.description 'Build a dictionary from treebank data'
+              f.action { |args, options| process_dictionary(args, options) }
+            end
+
+            c.command(:dictionaries) do |f|
+              f.syntax '[filename(s)]'
+              f.description 'Build multiple dictionaries (one per language) from treebank data'
+              f.action { |args, options| process_dictionaries(args, options) }
+            end
+
+            c.action do |_, _|
+              STDERR.puts 'Missing or invalid format. Use --help for more information.'
+              exit 1
+            end
+          end
+        end
+
+        def process_dictionary(args, options)
+          if args.empty?
+            STDERR.puts 'Missing output filename. Use --help for more information.'
+            exit 1
+          end
+
+          output_filename, *input_filenames = args
+
+          tb = PROIEL::Treebank.new
+          dict = PROIEL::Dictionary::Builder.new
+
+          if input_filenames.empty?
+            STDERR.puts 'Reading from standard input...'.green if options['verbose']
+
+            tb.load_from_xml(STDIN)
+            tb.sources.each { |source| dict.add_source!(source) }
+          else
+            input_filenames.each do |filename|
+              STDERR.puts "Reading #{filename}...".green if options['verbose']
+
+              tb.load_from_xml(filename)
+            end
+          end
+
+          tb.sources.each { |source| dict.add_source!(source) }
+
+          File.open(output_filename, 'w') do |f|
+            dict.to_xml(f)
+          end
+        end
+
+        def process_dictionaries(args, options)
+          dicts = {}
+
+          if args.empty?
+            STDERR.puts 'Reading from standard input...'.green if options['verbose']
+
+            tb = PROIEL::Treebank.new
+            tb.load_from_xml(STDIN)
+          else
+            tb = PROIEL::Treebank.new
+
+            args.each do |filename|
+              STDERR.puts "Reading #{filename}...".green if options['verbose']
+              tb.load_from_xml(filename)
+            end
+          end
+
+          tb.sources.each do |source|
+            dicts[source.language] ||= PROIEL::Dictionary::Builder.new
+            dicts[source.language].add_source!(source)
+          end
+
+          dicts.each do |language, dict|
+            File.open("#{language}.xml", 'w') do |f|
+              dict.to_xml(f)
+            end
+          end 
+        end
+      end
+    end
+  end
+end

data/lib/proiel/cli/commands/convert.rb

@@ -12,6 +12,8 @@ module PROIEL
               f.description 'Convert to PROIEL XML format'
               f.option 'remove-not-annotated', '--remove-not-annotated', 'Remove sentences that have not been annotated'
               f.option 'remove-not-reviewed', '--remove-not-reviewed', 'Remove sentences that have not been reviewed'
+              f.option 'remove-annotated', '--remove-annotated', 'Remove sentences that have been annotated'
+              f.option 'remove-reviewed', '--remove-reviewed', 'Remove sentences that have been reviewed'
               f.option 'remove-morphology', '--remove-morphology', 'Remove morphological annotation (part of speech, morphology and lemma)'
               f.option 'remove-syntax', '--remove-syntax', 'Remove syntactic annotation (relation, head ID and slashes)'
               f.option 'remove-information-structure', '--remove-information-structure', 'Remove informtion structure annotation (antecedent ID, information status and contrast group)'
@@ -20,13 +22,16 @@ module PROIEL
               f.option 'remove-annotator', '--remove-annotator', 'Remove annotator information'
               f.option 'remove-reviewer', '--remove-reviewer', 'Remove reviewer information'
               f.option 'remove-empty-divs', '--remove-empty-divs', 'Remove div elements that do not contain any sentences'
+              f.option 'infer-alignments', '--infer-alignments', 'Add inferred alignments when possible'
+              f.option 'remove-unaligned-sources', '--remove-unaligned-sources', 'Remove sources that are not aligned'
               f.action { |args, options| process(args, options, PROIEL::Converter::PROIELXML) }
             end
 
             c.command(:tnt) do |f|
               f.syntax '[options] filename(s)'
               f.description 'Convert to TNT/hunpos format'
-              f.option 'morphology', '-m', '--morphology', 'Include morphological tags'
+              f.option 'morphology', '-m', '--morphology', 'Include POS and morphological tags'
+              f.option 'pos', '-p', '--pos', 'Include POS tags'
               f.action { |args, options| process(args, options, PROIEL::Converter::TNT) }
             end
 
@@ -79,7 +84,7 @@ module PROIEL
           tb = PROIEL::Treebank.new
 
           if args.empty?
-            STDERR.puts "Reading from standard input...".green if options['verbose']
+            STDERR.puts 'Reading from standard input...'.green if options['verbose']
             tb.load_from_xml(STDIN)
           else
             args.each do |filename|

data/lib/proiel/cli/commands/dictionary.rb

@@ -0,0 +1,46 @@
+module PROIEL
+  module Commands
+    class Dictionary < Command
+      class << self
+        def init_with_program(prog)
+          prog.command(:dictionary) do |c|
+            c.syntax 'dictionary [options] filename(s)'
+            c.description 'Build a dictionary'
+
+            c.option 'glosses', '--merge-glosses glosses.tsv', 'Merge glosses from an external file'
+            c.option 'gloss-languages', '--merge-gloss-languages eng,rus', 'Merge glosses from selected languages'
+
+            c.action { |args, options| process(args, options) }
+          end
+        end
+
+        def process(args, options)
+          tb = PROIEL::Treebank.new
+          dict = PROIEL::DictionaryBuilder.new
+
+          args.each do |filename|
+            STDERR.puts "Reading #{filename}...".green if options['verbose']
+
+            tb.load_from_xml(filename)
+          end
+
+          if options['glosses']
+            languages = (options['gloss-languages'] || 'eng').split(',').map(&:to_sym)
+            if File.exist?(options['glosses'])
+              dict.add_external_glosses!(options['glosses'], languages)
+            else
+              STDERR.puts "#{options['glosses']} not found"
+              exit 1
+            end
+          end
+
+          tb.sources.each do |source|
+            dict.add_source!(source)
+          end
+
+          dict.to_xml(STDOUT)
+        end
+      end
+    end
+  end
+end

data/lib/proiel/cli/commands/info.rb

@@ -45,7 +45,7 @@ module PROIEL
             puts "   Printed text: #{pretty_printed_text_info(source)}"
             puts "   Electr. text: #{pretty_electronic_text_info(source)}"
             puts "   Size:         #{n} sentence(s), #{s.token_count} token(s)"
-            puts "   Annotation:   %.2f%% reviewed, %.2f%% annotated" % [r, a]
+            puts '   Annotation:   %.2f%% reviewed, %.2f%% annotated' % [r, a]
           end
         end
 

data/lib/proiel/cli/commands/shell.rb

@@ -0,0 +1,34 @@
+module PROIEL
+  module Commands
+    class Shell < Command
+      class << self
+        def init_with_program(prog)
+          prog.command(:shell) do |c|
+            c.syntax 'shell filename(s)'
+            c.description 'Launch a shell with the treebank loaded'
+
+            c.action do |args, options|
+              if args.empty?
+                STDERR.puts 'Missing filename(s). Use --help for more information.'
+              else
+                process(args, options)
+              end
+            end
+          end
+        end
+
+        def process(args, options)
+          tb = PROIEL::Treebank.new
+
+          args.each do |filename|
+            STDERR.puts "Reading #{filename}...".green if options['verbose']
+
+            tb.load_from_xml(filename)
+          end
+
+          binding.pry
+        end
+      end
+    end
+  end
+end

data/lib/proiel/cli/commands/tokenize.rb

@@ -12,7 +12,7 @@ module PROIEL
           end
         end
 
-        def process(args, options)
+        def process(args, _)
           if args.empty?
             STDERR.puts 'Missing filename. Use --help for more information.'
             exit 1
@@ -143,7 +143,7 @@ module PROIEL
         def read_body(f)
           f.rewind
 
-          Array.new.tap do |bdy|
+          [].tap do |bdy|
             f.each_line do |l|
               case l
               when /^%/

data/lib/proiel/cli/commands/validate.rb

@@ -10,7 +10,9 @@ module PROIEL
           end
         end
 
-        def process(args, options)
+        def process(args, _)
+          exit_code = 0
+
           if args.empty?
             STDERR.puts 'Missing filename(s). Use --help for more information.'
             exit 1
@@ -23,8 +25,6 @@ module PROIEL
 
             if v.valid?
               puts "#{filename} is valid".green
-
-              exit 0
             else
               puts "#{filename} is invalid".red
 
@@ -32,9 +32,11 @@ module PROIEL
                 puts "* #{error}"
               end
 
-              exit 1
+              exit_code = 1
             end
           end
+
+          exit exit_code
         end
       end
     end

data/lib/proiel/cli/commands/visualize.rb

@@ -4,40 +4,42 @@ module PROIEL
       class << self
         def init_with_program(prog)
           prog.command(:visualize) do |c|
-            c.syntax 'visualize sentences|divs|sources FILENAME(S)'
+            c.syntax 'visualize [OPTION(S)] FILENAME(S)'
             c.description 'Visualize treebank graphs'
-            c.option 'objects', '--objects sentences|divs|sources', 'Objects to visualize (default: sentences)'
+            c.option 'objects', '--objects sentences|divs|sources|SENTENCE-ID', 'Objects to visualize (default: sentences)'
             c.option 'format', '--format png|svg|dot', 'Output format (default: svg)'
-            c.option 'layout', '--layout classic|linearized|packed', 'Graph layout (default: classic)'
+            c.option 'layout', '--layout classic|linearized|packed|modern', 'Graph layout (default: classic)'
 
             c.action { |args, options| process(args, options) }
           end
         end
 
+        LAYOUTS = %w(classic linearized packed modern)
+
         def process(args, options)
           objects = options['objects'] || 'sentences'
           format = options['format'] || 'svg'
           layout = options['layout'] || 'classic'
 
-          if layout != 'classic' and layout != 'linearized' and layout != 'packed'
-            STDERR.puts "Invalid layout"
+          unless LAYOUTS.include?(layout)
+            STDERR.puts 'Invalid layout'
             exit 1
           end
 
-          if objects != 'sentences' and objects != 'divs' and objects != 'sources'
-            STDERR.puts "Invalid object type"
+          if objects != 'sentences' and objects != 'divs' and objects != 'sources' and objects.to_i.to_s != objects
+            STDERR.puts 'Invalid object type'
             exit 1
           end
 
           if format != 'png' and format != 'svg' and format != 'dot'
-            STDERR.puts "Invalid format"
+            STDERR.puts 'Invalid format'
             exit 1
           end
 
           tb = PROIEL::Treebank.new
 
           if args.empty?
-            STDERR.puts "Reading from standard input...".green if options['verbose']
+            STDERR.puts 'Reading from standard input...'.green if options['verbose']
             tb.load_from_xml(STDIN)
           else
             args.each do |filename|
@@ -50,14 +52,15 @@ module PROIEL
           tb.sources.each do |source|
             case objects
             when 'sources'
-              puts "This can take a very, very long time... Be patient!"
+              puts 'This can take a very, very long time... Be patient!'
               save_graph layout, format, source
             when 'divs'
               save_graphs source.divs, layout, format, source.id, source.divs.count
             when 'sentences'
               save_graphs source.sentences, layout, format, source.id, source.sentences.count
             else
-              raise
+              object = tb.find_sentence(objects.to_i)
+              save_graph(layout, format, object) if object
             end
           end
         end