proiel-cli 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 624e54e940c5251ec05abe8e970920b97535ca82
-  data.tar.gz: '09ca4feb9f1a38c8210a2571b809d2fd96a72b76'
+SHA256:
+  metadata.gz: 7f03b2148dc4a7dff5176d490e8850507b4984df13dafd56b04dd81763708df7
+  data.tar.gz: 8e41729b4ab79c7f48c8b34f77ac65a58188e950879c473f41dcc1c74af17d4d
 SHA512:
-  metadata.gz: 21a94b5606488a66c0cac245fd0ee1e922c3f3b1cb77f96934f7dd58a0bef00378d4f52bc3e5a6dec9d35a1b1228a8ea6b5eaf93a1d9aa6badec899b7e5d4158
-  data.tar.gz: '08f04cc516c5224447c3cf63da612a0d19c56fc02e6a2bb1c5c0fcf6b01cdc317aa5e613b3a06dfb11455d553d831f35aec1b1bb69e0f3e95f6ab9d9b18ce7c7'
+  metadata.gz: 838988757f5ee2360496297516047fe2715da30c8327aa411b6d3f395284d02a33123c263c238874e596aedcb53bbe9aeb79d63363fd712cb3480357c54bf87a
+  data.tar.gz: 1f629b9b036776f23a0c80851ea80f69082e8f2df8675ed268e0afdc86ac2e94264fecbaedd686a087475cc29e2abb7d99b090a4179c48c62516c402ea5179b5

data/README.md

@@ -11,7 +11,7 @@ This is a command-line interface for manipulating PROIEL treebanks.
 
 ## Installation
 
-This library requires Ruby >= 2.1. Install as
+This library requires Ruby >= 2.4. Install as
 
 ```shell
 gem install proiel-cli
@@ -42,8 +42,16 @@ Bug reports and pull requests are welcome on [GitHub](https://github.com/proiel/
 
 ## Development
 
-To contribute to development, check out the git repository from [GitHub](https://github.com/proiel/proiel-cli) and run `bin/setup` to install all development dependencies. Then run `rake` to run the tests.
+To contribute to development, check out the git repository from [GitHub](https://github.com/proiel/proiel-cli) and run `bundle install` to install all development dependencies. Then run `rake` to run the tests.
 
 To install a development version of this gem, run `bundle exec rake install`.
 
-To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the gem to [rubygems.org](https://rubygems.org).
+### Releasing a new version
+
+To release a new version:
+
+1.  Update the version number in `lib/proiel/cli/version.rb`.
+2.  Run `bundle exec rake release`. This will:
+    *   Create a git tag for the version.
+    *   Push git commits and tags to the remote repository.
+    *   Push the `.gem` file to [rubygems.org](https://rubygems.org).

data/bin/proiel

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 STDOUT.sync = true
 
-$:.unshift File.join(File.dirname(__FILE__), *%w{ .. lib })
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'proiel/cli'
 

data/lib/proiel/cli/commands/build.rb

@@ -0,0 +1,91 @@
+module PROIEL
+  module Commands
+    class Build < Command
+      class << self
+        def init_with_program(prog)
+          prog.command(:build) do |c|
+            c.syntax 'build resource'
+            c.description 'Build a derived resource'
+
+            c.command(:dictionary) do |f|
+              f.syntax 'output_filename [filename(s)]'
+              f.description 'Build a dictionary from treebank data'
+              f.action { |args, options| process_dictionary(args, options) }
+            end
+
+            c.command(:dictionaries) do |f|
+              f.syntax '[filename(s)]'
+              f.description 'Build multiple dictionaries (one per language) from treebank data'
+              f.action { |args, options| process_dictionaries(args, options) }
+            end
+
+            c.action do |_, _|
+              STDERR.puts 'Missing or invalid format. Use --help for more information.'
+              exit 1
+            end
+          end
+        end
+
+        def process_dictionary(args, options)
+          if args.empty?
+            STDERR.puts 'Missing output filename. Use --help for more information.'
+            exit 1
+          end
+
+          output_filename, *input_filenames = args
+
+          tb = PROIEL::Treebank.new
+          dict = PROIEL::Dictionary::Builder.new
+
+          if input_filenames.empty?
+            STDERR.puts 'Reading from standard input...'.green if options['verbose']
+
+            tb.load_from_xml(STDIN)
+            tb.sources.each { |source| dict.add_source!(source) }
+          else
+            input_filenames.each do |filename|
+              STDERR.puts "Reading #{filename}...".green if options['verbose']
+
+              tb.load_from_xml(filename)
+            end
+          end
+
+          tb.sources.each { |source| dict.add_source!(source) }
+
+          File.open(output_filename, 'w') do |f|
+            dict.to_xml(f)
+          end
+        end
+
+        def process_dictionaries(args, options)
+          dicts = {}
+
+          if args.empty?
+            STDERR.puts 'Reading from standard input...'.green if options['verbose']
+
+            tb = PROIEL::Treebank.new
+            tb.load_from_xml(STDIN)
+          else
+            tb = PROIEL::Treebank.new
+
+            args.each do |filename|
+              STDERR.puts "Reading #{filename}...".green if options['verbose']
+              tb.load_from_xml(filename)
+            end
+          end
+
+          tb.sources.each do |source|
+            dicts[source.language] ||= PROIEL::Dictionary::Builder.new
+            dicts[source.language].add_source!(source)
+          end
+
+          dicts.each do |language, dict|
+            File.open("#{language}.xml", 'w') do |f|
+              dict.to_xml(f)
+            end
+          end 
+        end
+      end
+    end
+  end
+end

data/lib/proiel/cli/commands/convert.rb

@@ -12,6 +12,8 @@ module PROIEL
               f.description 'Convert to PROIEL XML format'
               f.option 'remove-not-annotated', '--remove-not-annotated', 'Remove sentences that have not been annotated'
               f.option 'remove-not-reviewed', '--remove-not-reviewed', 'Remove sentences that have not been reviewed'
+              f.option 'remove-annotated', '--remove-annotated', 'Remove sentences that have been annotated'
+              f.option 'remove-reviewed', '--remove-reviewed', 'Remove sentences that have been reviewed'
               f.option 'remove-morphology', '--remove-morphology', 'Remove morphological annotation (part of speech, morphology and lemma)'
               f.option 'remove-syntax', '--remove-syntax', 'Remove syntactic annotation (relation, head ID and slashes)'
               f.option 'remove-information-structure', '--remove-information-structure', 'Remove informtion structure annotation (antecedent ID, information status and contrast group)'
@@ -20,13 +22,16 @@ module PROIEL
               f.option 'remove-annotator', '--remove-annotator', 'Remove annotator information'
               f.option 'remove-reviewer', '--remove-reviewer', 'Remove reviewer information'
               f.option 'remove-empty-divs', '--remove-empty-divs', 'Remove div elements that do not contain any sentences'
+              f.option 'infer-alignments', '--infer-alignments', 'Add inferred alignments when possible'
+              f.option 'remove-unaligned-sources', '--remove-unaligned-sources', 'Remove sources that are not aligned'
               f.action { |args, options| process(args, options, PROIEL::Converter::PROIELXML) }
             end
 
             c.command(:tnt) do |f|
               f.syntax '[options] filename(s)'
               f.description 'Convert to TNT/hunpos format'
-              f.option 'morphology', '-m', '--morphology', 'Include morphological tags'
+              f.option 'morphology', '-m', '--morphology', 'Include POS and morphological tags'
+              f.option 'pos', '-p', '--pos', 'Include POS tags'
               f.action { |args, options| process(args, options, PROIEL::Converter::TNT) }
             end
 
@@ -79,7 +84,7 @@ module PROIEL
           tb = PROIEL::Treebank.new
 
           if args.empty?
-            STDERR.puts "Reading from standard input...".green if options['verbose']
+            STDERR.puts 'Reading from standard input...'.green if options['verbose']
             tb.load_from_xml(STDIN)
           else
             args.each do |filename|

data/lib/proiel/cli/commands/dictionary.rb

@@ -0,0 +1,46 @@
+module PROIEL
+  module Commands
+    class Dictionary < Command
+      class << self
+        def init_with_program(prog)
+          prog.command(:dictionary) do |c|
+            c.syntax 'dictionary [options] filename(s)'
+            c.description 'Build a dictionary'
+
+            c.option 'glosses', '--merge-glosses glosses.tsv', 'Merge glosses from an external file'
+            c.option 'gloss-languages', '--merge-gloss-languages eng,rus', 'Merge glosses from selected languages'
+
+            c.action { |args, options| process(args, options) }
+          end
+        end
+
+        def process(args, options)
+          tb = PROIEL::Treebank.new
+          dict = PROIEL::DictionaryBuilder.new
+
+          args.each do |filename|
+            STDERR.puts "Reading #{filename}...".green if options['verbose']
+
+            tb.load_from_xml(filename)
+          end
+
+          if options['glosses']
+            languages = (options['gloss-languages'] || 'eng').split(',').map(&:to_sym)
+            if File.exist?(options['glosses'])
+              dict.add_external_glosses!(options['glosses'], languages)
+            else
+              STDERR.puts "#{options['glosses']} not found"
+              exit 1
+            end
+          end
+
+          tb.sources.each do |source|
+            dict.add_source!(source)
+          end
+
+          dict.to_xml(STDOUT)
+        end
+      end
+    end
+  end
+end

data/lib/proiel/cli/commands/info.rb

@@ -45,7 +45,7 @@ module PROIEL
             puts "   Printed text: #{pretty_printed_text_info(source)}"
             puts "   Electr. text: #{pretty_electronic_text_info(source)}"
             puts "   Size:         #{n} sentence(s), #{s.token_count} token(s)"
-            puts "   Annotation:   %.2f%% reviewed, %.2f%% annotated" % [r, a]
+            puts '   Annotation:   %.2f%% reviewed, %.2f%% annotated' % [r, a]
           end
         end
 

data/lib/proiel/cli/commands/shell.rb

@@ -0,0 +1,34 @@
+module PROIEL
+  module Commands
+    class Shell < Command
+      class << self
+        def init_with_program(prog)
+          prog.command(:shell) do |c|
+            c.syntax 'shell filename(s)'
+            c.description 'Launch a shell with the treebank loaded'
+
+            c.action do |args, options|
+              if args.empty?
+                STDERR.puts 'Missing filename(s). Use --help for more information.'
+              else
+                process(args, options)
+              end
+            end
+          end
+        end
+
+        def process(args, options)
+          tb = PROIEL::Treebank.new
+
+          args.each do |filename|
+            STDERR.puts "Reading #{filename}...".green if options['verbose']
+
+            tb.load_from_xml(filename)
+          end
+
+          binding.pry
+        end
+      end
+    end
+  end
+end

data/lib/proiel/cli/commands/tokenize.rb

@@ -12,7 +12,7 @@ module PROIEL
           end
         end
 
-        def process(args, options)
+        def process(args, _)
           if args.empty?
             STDERR.puts 'Missing filename. Use --help for more information.'
             exit 1
@@ -143,7 +143,7 @@ module PROIEL
         def read_body(f)
           f.rewind
 
-          Array.new.tap do |bdy|
+          [].tap do |bdy|
             f.each_line do |l|
               case l
               when /^%/

data/lib/proiel/cli/commands/validate.rb

@@ -10,7 +10,7 @@ module PROIEL
           end
         end
 
-        def process(args, options)
+        def process(args, _)
           exit_code = 0
 
           if args.empty?

data/lib/proiel/cli/commands/visualize.rb

@@ -4,40 +4,42 @@ module PROIEL
       class << self
         def init_with_program(prog)
           prog.command(:visualize) do |c|
-            c.syntax 'visualize sentences|divs|sources FILENAME(S)'
+            c.syntax 'visualize [OPTION(S)] FILENAME(S)'
             c.description 'Visualize treebank graphs'
-            c.option 'objects', '--objects sentences|divs|sources', 'Objects to visualize (default: sentences)'
+            c.option 'objects', '--objects sentences|divs|sources|SENTENCE-ID', 'Objects to visualize (default: sentences)'
             c.option 'format', '--format png|svg|dot', 'Output format (default: svg)'
-            c.option 'layout', '--layout classic|linearized|packed', 'Graph layout (default: classic)'
+            c.option 'layout', '--layout classic|linearized|packed|modern', 'Graph layout (default: classic)'
 
             c.action { |args, options| process(args, options) }
           end
         end
 
+        LAYOUTS = %w(classic linearized packed modern)
+
         def process(args, options)
           objects = options['objects'] || 'sentences'
           format = options['format'] || 'svg'
           layout = options['layout'] || 'classic'
 
-          if layout != 'classic' and layout != 'linearized' and layout != 'packed'
-            STDERR.puts "Invalid layout"
+          unless LAYOUTS.include?(layout)
+            STDERR.puts 'Invalid layout'
             exit 1
           end
 
-          if objects != 'sentences' and objects != 'divs' and objects != 'sources'
-            STDERR.puts "Invalid object type"
+          if objects != 'sentences' and objects != 'divs' and objects != 'sources' and objects.to_i.to_s != objects
+            STDERR.puts 'Invalid object type'
             exit 1
           end
 
           if format != 'png' and format != 'svg' and format != 'dot'
-            STDERR.puts "Invalid format"
+            STDERR.puts 'Invalid format'
             exit 1
           end
 
           tb = PROIEL::Treebank.new
 
           if args.empty?
-            STDERR.puts "Reading from standard input...".green if options['verbose']
+            STDERR.puts 'Reading from standard input...'.green if options['verbose']
             tb.load_from_xml(STDIN)
           else
             args.each do |filename|
@@ -50,14 +52,15 @@ module PROIEL
           tb.sources.each do |source|
             case objects
             when 'sources'
-              puts "This can take a very, very long time... Be patient!"
+              puts 'This can take a very, very long time... Be patient!'
               save_graph layout, format, source
             when 'divs'
               save_graphs source.divs, layout, format, source.id, source.divs.count
             when 'sentences'
               save_graphs source.sentences, layout, format, source.id, source.sentences.count
             else
-              raise
+              object = tb.find_sentence(objects.to_i)
+              save_graph(layout, format, object) if object
             end
           end
         end