opener-tokenizer 1.1.2 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f9af1f4a79564c201746a78973f5fc3cdfe272f1
-  data.tar.gz: 41e00e0819e7a16aa5d9e509fb4f28751811b292
+SHA256:
+  metadata.gz: 5d249d10dfdca563615ea4ee70e905965970e36bca57ba78075c333ead37fad5
+  data.tar.gz: feb7d682b0c96c765a6bfee38178223ccf9e4c4d05ef0ad40d1a66712a28ee2a
 SHA512:
-  metadata.gz: 563674a5a7f855eff87bd6659a5688f35502c0790fbde1797dc1b27f8f45f03f4c42ca8da2dfbe501da6c5a8db7523bc4cd308adf765c0e4a754e3874f1563c8
-  data.tar.gz: e716377afcbc496894a89b95aeec064c89402c60839fb1213dfce2990a2baaab083e0f46950a1d8791c54a1028ac4c7faefd00dd49ee36b463af90204b55602d
+  metadata.gz: 8e48843449d69c2acea361f285d76c0d4b73887a497fa88146e84456c21d76ce1518c5c1d435e67301c7443c9d36d2f3b8fecd96c004dad1f91d29e627092be4
+  data.tar.gz: 5af4a2af84e13237345fa69bfd8908b8df653abc53fcd3fc32a727efa5399a1c7499c2ad5061c981b5ed6069e0b3bca9fadd6b164c2190e219026b6abb7716c6

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright 2014 OpeNER Project Consortium
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -1,11 +1,15 @@
 Introduction
 ------------
 
-The tokenizer tokenizes a text into sentences and words. 
+The tokenizer tokenizes a text into sentences and words.
 
 ### Confused by some terminology?
 
-This software is part of a larger collection of natural language processing tools known as "the OpeNER project". You can find more information about the project at [the OpeNER portal](http://opener-project.github.io). There you can also find references to terms like KAF (an XML standard to represent linguistic annotations in texts), component, cores, scenario's and pipelines.
+This software is part of a larger collection of natural language processing
+tools known as "the OpeNER project". You can find more information about the
+project at [the OpeNER portal](http://opener-project.github.io). There you can
+also find references to terms like KAF (an XML standard to represent linguistic
+annotations in texts), component, cores, scenario's and pipelines.
 
 Quick Use Example
 -----------------
@@ -20,13 +24,14 @@ output KAF by default.
 
 ### Command line interface
 
-You should now be able to call the tokenizer as a regular shell
-command: by its name. Once installed the gem normally sits in your path so you can call it directly from anywhere.
+You should now be able to call the tokenizer as a regular shell command: by its
+name. Once installed the gem normally sits in your path so you can call it
+directly from anywhere.
 
 Tokenizing some text:
 
     echo "This is English text" | tokenizer -l en --no-kaf
-    
+
 Will result in
 
     <?xml version="1.0" encoding="UTF-8" standalone="no"?>
@@ -45,11 +50,13 @@ Will result in
       </text>
     </KAF>
 
-  The available languages for tokenization are: English (en), German (de), Dutch (nl), French (fr), Spanish (es), Italian (it)
+The available languages for tokenization are: English (en), German (de), Dutch
+(nl), French (fr), Spanish (es), Italian (it)
 
 #### KAF input format
 
-The tokenizer is capable of taking KAF as input, and actually does so by default. You can do so like this:
+The tokenizer is capable of taking KAF as input, and actually does so by
+default. You can do so like this:
 
     echo "<?xml version='1.0' encoding='UTF-8' standalone='no'?><KAF version='v1.opener' xml:lang='en'><raw>This is what I call, a test!</raw></KAF>" | tokenizer
 
@@ -72,7 +79,8 @@ Will result in
       </text>
     </KAF>
 
-If the argument -k (--kaf) is passed, then the argument -l (--language) is ignored.
+If the argument -k (--kaf) is passed, then the argument -l (--language) is
+ignored.
 
 ### Webservices
 
@@ -80,7 +88,8 @@ You can launch a language identification webservice by executing:
 
     tokenizer-server
 
-This will launch a mini webserver with the webservice. It defaults to port 9292, so you can access it at <http://localhost:9292>.
+This will launch a mini webserver with the webservice. It defaults to port 9292,
+so you can access it at <http://localhost:9292>.
 
 To launch it on a different port provide the `-p [port-number]` option like this:
 
@@ -88,19 +97,25 @@ To launch it on a different port provide the `-p [port-number]` option like this
 
 It then launches at <http://localhost:1234>
 
-Documentation on the Webservice is provided by surfing to the urls provided above. For more information on how to launch a webservice run the command with the ```-h``` option.
+Documentation on the Webservice is provided by surfing to the urls provided
+above. For more information on how to launch a webservice run the command with
+the `--help` option.
 
 
 ### Daemon
 
-Last but not least the tokenizer comes shipped with a daemon that can read jobs (and write) jobs to and from Amazon SQS queues. For more information type:
+Last but not least the tokenizer comes shipped with a daemon that can read jobs
+(and write) jobs to and from Amazon SQS queues. For more information type:
 
-    tokenizer-daemon -h
+    tokenizer-daemon --help
 
 Description of dependencies
 ---------------------------
 
-This component runs best if you run it in an environment suited for OpeNER components. You can find an installation guide and helper tools in the [OpeNER installer](https://github.com/opener-project/opener-installer) and [an installation guide on the Opener Website](http://opener-project.github.io/getting-started/how-to/local-installation.html)
+This component runs best if you run it in an environment suited for OpeNER
+components. You can find an installation guide and helper tools in the
+[OpeNER installer](https://github.com/opener-project/opener-installer) and
+[an installation guide on the Opener Website](http://opener-project.github.io/getting-started/how-to/local-installation.html).
 
 At least you need the following system setup:
 
@@ -113,16 +128,20 @@ At least you need the following system setup:
 
 * Maven (for building the Gem)
 
-
 Language Extension
 ------------------
 
-The tokenizer module is a wrapping around a Perl script, which performs the actual tokenization based on rules (when to break a character sequence). The tokenizer already supports a lot of languages. Have a look to the core script to figure out how to extend to new languages.
+The tokenizer module is a wrapping around a Perl script, which performs the
+actual tokenization based on rules (when to break a character sequence). The
+tokenizer already supports a lot of languages. Have a look to the core script to
+figure out how to extend to new languages.
 
 The Core
 --------
 
-The component is a fat wrapper around the actual language technology core. The core is a rule based tokenizer implemented in Perl. You can find the core technologies in the following repositories:
+The component is a fat wrapper around the actual language technology core. The
+core is a rule based tokenizer implemented in Perl. You can find the core
+technologies in the following repositories:
 
 * [tokenizer-base](http://github.com/opener-project/tokenizer-base)
 
@@ -135,9 +154,8 @@ Where to go from here
 Report problem/Get help
 -----------------------
 
-If you encounter problems, please email <support@opener-project.eu> or leave an issue in the 
-[issue tracker](https://github.com/opener-project/tokenizer/issues).
-
+If you encounter problems, please email <support@opener-project.eu> or leave an
+issue in the [issue tracker](https://github.com/opener-project/tokenizer/issues).
 
 Contributing
 ------------

data/bin/tokenizer

@@ -2,6 +2,6 @@
 
 require_relative '../lib/opener/tokenizer'
 
-cli = Opener::Tokenizer::CLI.new(:args => ARGV)
+cli = Opener::Tokenizer::CLI.new
 
-cli.run(STDIN.tty? ? nil : STDIN.read)
+cli.run

data/bin/tokenizer-daemon

@@ -1,9 +1,10 @@
 #!/usr/bin/env ruby
-#
-require 'rubygems'
+
 require 'opener/daemons'
 
-exec_path = File.expand_path("../../exec/tokenizer.rb", __FILE__)
-Opener::Daemons::Controller.new(:name=>"tokenizer",
-                                :exec_path=>exec_path)
+controller = Opener::Daemons::Controller.new(
+  :name      => 'opener-tokenizer',
+  :exec_path => File.expand_path('../../exec/tokenizer.rb', __FILE__)
+)
 
+controller.run

data/bin/tokenizer-server

@@ -1,8 +1,10 @@
 #!/usr/bin/env ruby
 
-require 'puma/cli'
+require 'opener/webservice'
 
-rack_config = File.expand_path('../../config.ru', __FILE__)
+parser = Opener::Webservice::OptionParser.new(
+  'opener-tokenizer',
+  File.expand_path('../../config.ru', __FILE__)
+)
 
-cli = Puma::CLI.new([rack_config] + ARGV)
-cli.run
+parser.run

data/exec/tokenizer.rb

@@ -1,8 +1,9 @@
 #!/usr/bin/env ruby
-#
+
 require 'opener/daemons'
-require 'opener/tokenizer'
 
-options = Opener::Daemons::OptParser.parse!(ARGV)
-daemon = Opener::Daemons::Daemon.new(Opener::Tokenizer, options)
+require_relative '../lib/opener/tokenizer'
+
+daemon = Opener::Daemons::Daemon.new(Opener::Tokenizer)
+
 daemon.start

data/lib/opener/tokenizer.rb

@@ -1,8 +1,9 @@
-require 'opener/tokenizers/base'
-require 'nokogiri'
 require 'open3'
-require 'optparse'
+
 require 'opener/core'
+require 'opener/tokenizers/base'
+require 'nokogiri'
+require 'slop'
 
 require_relative 'tokenizer/version'
 require_relative 'tokenizer/cli'
@@ -41,8 +42,10 @@ module Opener
     #
     # @option options [Array] :args Collection of arbitrary arguments to pass
     #  to the individual tokenizer commands.
+    #
     # @option options [String] :language The language to use for the
     #  tokenization process.
+    #
     # @option options [TrueClass|FalseClass] :kaf When set to `true` the input
     #  is assumed to be KAF.
     #
@@ -51,39 +54,36 @@ module Opener
     end
 
     ##
-    # Processes the input and returns an array containing the output of STDOUT,
-    # STDERR and an object containing process information.
+    # Tokenizes the input and returns the results as a KAF document.
     #
     # @param [String] input
-    # @return [Array]
+    # @return [String]
     #
-    def run(input)
-      begin
-        if options[:kaf]
-          language, input = kaf_elements(input)
-        else
-          language = options[:language]
-        end
-        
-        unless valid_language?(language)
-          raise ArgumentError, "The specified language (#{language}) is invalid"
-        end
-        
-        kernel = language_constant(language).new(:args => options[:args])
-        
-        stdout, stderr, process = Open3.capture3(*kernel.command.split(" "), :stdin_data => input)
-        raise stderr unless process.success?
-        return stdout
-        
-      rescue Exception => error
-        return Opener::Core::ErrorLayer.new(input, error.message, self.class).add
+    def run input, params = {}
+      if options[:kaf]
+        language, input = kaf_elements(input)
+      else
+        language = options[:language]
+      end
+
+      unless valid_language?(language)
+        raise Core::UnsupportedLanguageError, language
       end
+
+      kernel = language_constant(language).new(:args => options[:args])
+
+      stdout, stderr, process = Open3.capture3(
+        *kernel.command.split(" "),
+        :stdin_data => input
+      )
+
+      raise stderr unless process.success?
+
+      return stdout
     end
 
     alias tokenize run
 
-    private
-
     ##
     # Returns an Array containing the language an input from a KAF document.
     #
@@ -98,19 +98,25 @@ module Opener
       return language, text
     end
 
+    private
+
     ##
     # @param [String] language
     # @return [Class]
     #
     def language_constant(language)
-      Opener::Tokenizers.const_get(language.upcase)
+      name = Core::LanguageCode.constant_name(language)
+
+      Tokenizers.const_get(name)
     end
 
     ##
     # @return [TrueClass|FalseClass]
     #
     def valid_language?(language)
-      return Opener::Tokenizers.const_defined?(language.upcase)
+      name = Core::LanguageCode.constant_name(language)
+
+      return Tokenizers.const_defined?(name)
     end
   end # Tokenizer
 end # Opener

data/lib/opener/tokenizer/cli.rb

@@ -1,110 +1,92 @@
 module Opener
   class Tokenizer
     ##
-    # CLI wrapper around {Opener::Tokenizer} using OptionParser.
+    # CLI wrapper around {Opener::Tokenizer} using Slop.
     #
-    # @!attribute [r] options
-    #  @return [Hash]
-    # @!attribute [r] option_parser
-    #  @return [OptionParser]
+    # @!attribute [r] parser
+    #  @return [Slop]
     #
     class CLI
-      attr_reader :options, :option_parser
+      attr_reader :parser
+
+      def initialize
+        @parser = configure_slop
+      end
 
       ##
-      # @param [Hash] options
+      # @param [Array] argv
       #
-      def initialize(options = {})
-        @options = DEFAULT_OPTIONS.merge(options)
-
-        @option_parser = OptionParser.new do |opts|
-          opts.program_name   = 'tokenizer'
-          opts.summary_indent = '  '
-
-          opts.on('-h', '--help', 'Shows this help message') do
-            show_help
-          end
-
-          opts.on('-v', '--version', 'Shows the current version') do
-            show_version
-          end
+      def run(argv = ARGV)
+        parser.parse(argv)
+      end
 
-          opts.on(
-            '-l',
-            '--language [VALUE]',
-            'Uses this specific language'
-          ) do |value|
-            @options[:language] = value
-            @options[:kaf] = false
-          end
+      ##
+      # @return [Slop]
+      #
+      def configure_slop
+        return Slop.new(:strict => false, :indent => 2, :help => true) do
+          banner 'Usage: tokenizer [OPTIONS]'
 
-          opts.on('-k', '--kaf', 'Treats the input as a KAF document') do
-            @options[:kaf] = true
-          end
+          separator <<-EOF.chomp
 
-          opts.on('-p', '--plain', 'Treats the input as plain text') do
-            @options[:kaf] = false
-          end
+About:
 
-          opts.separator <<-EOF
+    Tokenizer for KAF/plain text documents with support for various languages
+    such as Dutch and English. This command reads input from STDIN.
 
 Examples:
 
-  cat example.txt | #{opts.program_name} -l en # Manually specify the language
-  cat example.kaf | #{opts.program_name}       # Uses the xml:lang attribute
+    cat example.txt | tokenizer -l en # Manually specify the language
+    cat example.kaf | tokenizer       # Uses the xml:lang attribute
 
 Languages:
 
-  * Dutch (nl)
-  * English (en)
-  * French (fr)
-  * German (de)
-  * Italian (it)
-  * Spanish (es)
+    * Dutch (nl)
+    * English (en)
+    * French (fr)
+    * German (de)
+    * Italian (it)
+    * Spanish (es)
 
 KAF Input:
 
-  If you give a KAF file as an input (-k or --kaf) the language is taken from
-  the xml:lang attribute inside the file. Else it expects that you give the
-  language as an argument (-l or --language)
+    If you give a KAF file as an input (-k or --kaf) the language is taken from
+    the xml:lang attribute inside the file. Else it expects that you give the
+    language as an argument (-l or --language)
 
-Sample KAF syntax:
+Example KAF:
 
-  <?xml version="1.0" encoding="UTF-8" standalone="no"?>
-  <KAF version="v1.opener" xml:lang="en">
-    <raw>This is some text.</raw>
-  </KAF>
+    <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+    <KAF version="v1.opener" xml:lang="en">
+      <raw>This is some text.</raw>
+    </KAF>
           EOF
-        end
-      end
 
-      ##
-      # @param [String] input
-      #
-      def run(input)
-        option_parser.parse!(options[:args])
+          separator "\nOptions:\n"
 
-        tokenizer = Tokenizer.new(options)
+          on :v, :version, 'Shows the current version' do
+            abort "tokenizer v#{VERSION} on #{RUBY_DESCRIPTION}"
+          end
 
-        stdout, stderr, process = tokenizer.run(input)
+          on :l=, :language=, 'A specific language to use',
+            :as      => String,
+            :default => DEFAULT_LANGUAGE
 
-        puts stdout
-      end
+          on :k, :kaf, 'Treats the input as a KAF document'
+          on :p, :plain, 'Treats the input as plain text'
 
-      private
+          run do |opts, args|
+            tokenizer = Tokenizer.new(
+              :args     => args,
+              :kaf      => opts[:plain] ? false : true,
+              :language => opts[:language]
+            )
 
-      ##
-      # Shows the help message and exits the program.
-      #
-      def show_help
-        abort option_parser.to_s
-      end
+            input = STDIN.tty? ? nil : STDIN.read
 
-      ##
-      # Shows the version and exits the program.
-      #
-      def show_version
-        abort "#{option_parser.program_name} v#{VERSION} on #{RUBY_DESCRIPTION}"
+            puts tokenizer.run(input)
+          end
+        end
       end
     end # CLI
   end # Tokenizer

data/lib/opener/tokenizer/server.rb

@@ -1,5 +1,3 @@
-require 'sinatra/base'
-require 'httpclient'
 require 'opener/webservice'
 
 module Opener
@@ -7,10 +5,11 @@ module Opener
     ##
     # Text tokenizer server powered by Sinatra.
     #
-    class Server < Webservice
+    class Server < Webservice::Server
       set :views, File.expand_path('../views', __FILE__)
-      text_processor Tokenizer
-      accepted_params :input, :kaf, :language
+
+      self.text_processor  = Tokenizer
+      self.accepted_params = [:input, :kaf, :language]
     end # Server
   end # Tokenizer
 end # Opener

data/lib/opener/tokenizer/version.rb

@@ -1,5 +1,7 @@
 module Opener
   class Tokenizer
-    VERSION = "1.1.2"
+
+    VERSION = '2.2.0'
+
   end
 end

data/opener-tokenizer.gemspec

@@ -7,7 +7,8 @@ Gem::Specification.new do |gem|
   gem.summary     = 'Gem that wraps up the the tokenizer cores'
   gem.description = gem.summary
   gem.homepage    = 'http://opener-project.github.com/'
-  gem.has_rdoc    = "yard"
+
+  gem.license = 'Apache 2.0'
 
   gem.required_ruby_version = '>= 1.9.2'
 
@@ -16,23 +17,21 @@ Gem::Specification.new do |gem|
     'lib/**/*',
     'config.ru',
     '*.gemspec',
-    'README.md'
+    'README.md',
+    'LICENSE.txt'
   ]).select { |file| File.file?(file) }
 
   gem.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
 
-  gem.add_dependency 'opener-tokenizer-base', '>= 0.3.1'
-  gem.add_dependency 'opener-webservice'
+  gem.add_dependency 'opener-tokenizer-base', '~> 1.0'
+  gem.add_dependency 'opener-webservice', '~> 2.1'
+  gem.add_dependency 'opener-daemons', '~> 2.1'
+  gem.add_dependency 'opener-core', '~> 2.4'
 
   gem.add_dependency 'nokogiri'
-  gem.add_dependency 'sinatra', '~>1.4.2'
-  gem.add_dependency 'httpclient'
-  gem.add_dependency 'opener-daemons'
-  gem.add_dependency 'opener-core', '>= 1.0.2'
-  gem.add_dependency 'puma'
+  gem.add_dependency 'slop', '~> 3.5'
 
   gem.add_development_dependency 'rspec'
-  gem.add_development_dependency 'cucumber'
   gem.add_development_dependency 'pry'
   gem.add_development_dependency 'rake'
 end

metadata

@@ -1,183 +1,141 @@
 --- !ruby/object:Gem::Specification
 name: opener-tokenizer
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 2.2.0
 platform: ruby
 authors:
 - development@olery.com
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-06-19 00:00:00.000000000 Z
+date: 2020-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: opener-tokenizer-base
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 0.3.1
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: '1.0'
+  name: opener-tokenizer-base
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: opener-webservice
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.1'
+  name: opener-webservice
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: nokogiri
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.1'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.1'
+  name: opener-daemons
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: sinatra
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.2
+        version: '2.1'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.2
+        version: '2.4'
+  name: opener-core
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: httpclient
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.4'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: nokogiri
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: opener-daemons
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: opener-core
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.0.2
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: '3.5'
+  name: slop
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: puma
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
-  type: :runtime
+        version: '3.5'
 - !ruby/object:Gem::Dependency
-  name: rspec
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: rspec
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: cucumber
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: pry
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: pry
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: rake
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :development
 description: Gem that wraps up the the tokenizer cores
 email:
 executables:
@@ -187,6 +145,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE.txt
 - README.md
 - bin/tokenizer
 - bin/tokenizer-daemon
@@ -202,7 +161,8 @@ files:
 - lib/opener/tokenizer/views/result.erb
 - opener-tokenizer.gemspec
 homepage: http://opener-project.github.com/
-licenses: []
+licenses:
+- Apache 2.0
 metadata: {}
 post_install_message:
 rdoc_options: []
@@ -210,17 +170,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.7.9
 signing_key:
 specification_version: 4
 summary: Gem that wraps up the the tokenizer cores