opener-tokenizer 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dab5882d292da38d032ad2de7dfb8c289372428b
-  data.tar.gz: 19ee04d381d370e64d606dc41e3472296980df67
+  metadata.gz: a01694d8c3c4cabbeadcee2b2e478ed699a21555
+  data.tar.gz: 3513b6bd6fe22ea36edc82eb2248204f24dedc8e
 SHA512:
-  metadata.gz: dcdb9e6f44524b5a1a23aa54dea0ed68e4b048b5b0b929c0964aeb576d2eecdc2877a0d054cd73e8afd67ad936016a5e044ae9b95a507219a9f930a7bf4775ec
-  data.tar.gz: e8660e048b3b58049bed277b207a69a5c763eac89049c251b750e0d4e111bb003030987540758686f175f4cfd2be76959a3534262faf57b5475f7982286a966c
+  metadata.gz: 3928380d43ccd980b675562c0a2b19e3ae45ebe410d300314685678b9c2829dac0c66b5cc805fbc4a22ed6267563869fe7fca5bee11ddc78e17cea6cabcec5f6
+  data.tar.gz: dc0a947729d4f97f49a919a240c8ab73b8122315add3aeecc65cd8ada3c0474e5faca2d4c0e84f69dce77ad74937f22e7b65932bae146c5592b129d17f5ed42f

data/bin/tokenizer

@@ -2,6 +2,6 @@
 
 require_relative '../lib/opener/tokenizer'
 
-cli = Opener::Tokenizer::CLI.new(:args => ARGV)
+cli = Opener::Tokenizer::CLI.new
 
-cli.run(STDIN.tty? ? nil : STDIN.read)
+cli.run

data/lib/opener/tokenizer.rb

@@ -1,7 +1,7 @@
 require 'opener/tokenizers/base'
 require 'nokogiri'
 require 'open3'
-require 'optparse'
+require 'slop'
 
 require_relative 'tokenizer/version'
 require_relative 'tokenizer/cli'
@@ -52,35 +52,32 @@ module Opener
     end
 
     ##
-    # Processes the input and returns an array containing the output of STDOUT,
-    # STDERR and an object containing process information.
+    # Tokenizes the input and returns the results as a KAF document.
     #
     # @param [String] input
-    # @return [Array]
+    # @return [String]
     #
     def run(input)
-      begin
-        if options[:kaf]
-          language, input = kaf_elements(input)
-        else
-          language = options[:language]
-        end
+      if options[:kaf]
+        language, input = kaf_elements(input)
+      else
+        language = options[:language]
+      end
 
-        unless valid_language?(language)
-          raise ArgumentError, "The specified language (#{language}) is invalid"
-        end
+      unless valid_language?(language)
+        raise ArgumentError, "The specified language (#{language}) is invalid"
+      end
 
-        kernel = language_constant(language).new(:args => options[:args])
+      kernel = language_constant(language).new(:args => options[:args])
 
-        stdout, stderr, process = Open3.capture3(
-          *kernel.command.split(" "),
-          :stdin_data => input
-        )
+      stdout, stderr, process = Open3.capture3(
+        *kernel.command.split(" "),
+        :stdin_data => input
+      )
 
-        raise stderr unless process.success?
+      raise stderr unless process.success?
 
-        return stdout
-      end
+      return stdout
     end
 
     alias tokenize run

data/lib/opener/tokenizer/cli.rb

@@ -1,110 +1,92 @@
 module Opener
   class Tokenizer
     ##
-    # CLI wrapper around {Opener::Tokenizer} using OptionParser.
+    # CLI wrapper around {Opener::Tokenizer} using Slop.
     #
-    # @!attribute [r] options
-    #  @return [Hash]
-    # @!attribute [r] option_parser
-    #  @return [OptionParser]
+    # @!attribute [r] parser
+    #  @return [Slop]
     #
     class CLI
-      attr_reader :options, :option_parser
+      attr_reader :parser
+
+      def initialize
+        @parser = configure_slop
+      end
 
       ##
-      # @param [Hash] options
+      # @param [Array] argv
       #
-      def initialize(options = {})
-        @options = DEFAULT_OPTIONS.merge(options)
-
-        @option_parser = OptionParser.new do |opts|
-          opts.program_name   = 'tokenizer'
-          opts.summary_indent = '  '
-
-          opts.on('-h', '--help', 'Shows this help message') do
-            show_help
-          end
-
-          opts.on('-v', '--version', 'Shows the current version') do
-            show_version
-          end
+      def run(argv = ARGV)
+        parser.parse(argv)
+      end
 
-          opts.on(
-            '-l',
-            '--language [VALUE]',
-            'Uses this specific language'
-          ) do |value|
-            @options[:language] = value
-            @options[:kaf] = false
-          end
+      ##
+      # @return [Slop]
+      #
+      def configure_slop
+        return Slop.new(:strict => false, :indent => 2, :help => true) do
+          banner 'Usage: tokenizer [OPTIONS]'
 
-          opts.on('-k', '--kaf', 'Treats the input as a KAF document') do
-            @options[:kaf] = true
-          end
+          separator <<-EOF.chomp
 
-          opts.on('-p', '--plain', 'Treats the input as plain text') do
-            @options[:kaf] = false
-          end
+About:
 
-          opts.separator <<-EOF
+    Tokenizer for KAF/plain text documents with support for various languages
+    such as Dutch and English. This command reads input from STDIN.
 
 Examples:
 
-  cat example.txt | #{opts.program_name} -l en # Manually specify the language
-  cat example.kaf | #{opts.program_name}       # Uses the xml:lang attribute
+    cat example.txt | tokenizer -l en # Manually specify the language
+    cat example.kaf | tokenizer       # Uses the xml:lang attribute
 
 Languages:
 
-  * Dutch (nl)
-  * English (en)
-  * French (fr)
-  * German (de)
-  * Italian (it)
-  * Spanish (es)
+    * Dutch (nl)
+    * English (en)
+    * French (fr)
+    * German (de)
+    * Italian (it)
+    * Spanish (es)
 
 KAF Input:
 
-  If you give a KAF file as an input (-k or --kaf) the language is taken from
-  the xml:lang attribute inside the file. Else it expects that you give the
-  language as an argument (-l or --language)
+    If you give a KAF file as an input (-k or --kaf) the language is taken from
+    the xml:lang attribute inside the file. Else it expects that you give the
+    language as an argument (-l or --language)
 
-Sample KAF syntax:
+Example KAF:
 
-  <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-  <KAF version="v1.opener" xml:lang="en">
-    <raw>This is some text.</raw>
-  </KAF>
+    <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+    <KAF version="v1.opener" xml:lang="en">
+      <raw>This is some text.</raw>
+    </KAF>
           EOF
-        end
-      end
 
-      ##
-      # @param [String] input
-      #
-      def run(input)
-        option_parser.parse!(options[:args])
+          separator "\nOptions:\n"
 
-        tokenizer = Tokenizer.new(options)
+          on :v, :version, 'Shows the current version' do
+            abort "tokenizer v#{VERSION} on #{RUBY_DESCRIPTION}"
+          end
 
-        stdout, stderr, process = tokenizer.run(input)
+          on :l=, :language=, 'A specific language to use',
+            :as      => String,
+            :default => DEFAULT_LANGUAGE
 
-        puts stdout
-      end
+          on :k, :kaf, 'Treats the input as a KAF document'
+          on :p, :plain, 'Treats the input as plain text'
 
-      private
+          run do |opts, args|
+            tokenizer = Tokenizer.new(
+              :args     => args,
+              :kaf      => opts[:plain] ? false : true,
+              :language => opts[:language]
+            )
 
-      ##
-      # Shows the help message and exits the program.
-      #
-      def show_help
-        abort option_parser.to_s
-      end
+            input = STDIN.tty? ? nil : STDIN.read
 
-      ##
-      # Shows the version and exits the program.
-      #
-      def show_version
-        abort "#{option_parser.program_name} v#{VERSION} on #{RUBY_DESCRIPTION}"
+            puts tokenizer.run(input)
+          end
+        end
       end
     end # CLI
   end # Tokenizer

data/lib/opener/tokenizer/version.rb

@@ -1,5 +1,5 @@
 module Opener
   class Tokenizer
-    VERSION = '2.0.0'
+    VERSION = '2.1.0'
   end
 end

data/opener-tokenizer.gemspec

@@ -24,12 +24,14 @@ Gem::Specification.new do |gem|
 
   gem.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
 
-  gem.add_dependency 'nokogiri'
   gem.add_dependency 'opener-tokenizer-base', '~> 1.0'
   gem.add_dependency 'opener-webservice', '~> 2.1'
   gem.add_dependency 'opener-daemons', '~> 2.1'
   gem.add_dependency 'opener-core', '~> 2.0'
 
+  gem.add_dependency 'nokogiri'
+  gem.add_dependency 'slop', '~> 3.5'
+
   gem.add_development_dependency 'rspec'
   gem.add_development_dependency 'cucumber'
   gem.add_development_dependency 'pry'

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: opener-tokenizer
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - development@olery.com
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-24 00:00:00.000000000 Z
+date: 2014-11-26 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: opener-tokenizer-base
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +66,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement