opener-tokenizer 1.1.2 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f9af1f4a79564c201746a78973f5fc3cdfe272f1
-  data.tar.gz: 41e00e0819e7a16aa5d9e509fb4f28751811b292
+  metadata.gz: dab5882d292da38d032ad2de7dfb8c289372428b
+  data.tar.gz: 19ee04d381d370e64d606dc41e3472296980df67
 SHA512:
-  metadata.gz: 563674a5a7f855eff87bd6659a5688f35502c0790fbde1797dc1b27f8f45f03f4c42ca8da2dfbe501da6c5a8db7523bc4cd308adf765c0e4a754e3874f1563c8
-  data.tar.gz: e716377afcbc496894a89b95aeec064c89402c60839fb1213dfce2990a2baaab083e0f46950a1d8791c54a1028ac4c7faefd00dd49ee36b463af90204b55602d
+  metadata.gz: dcdb9e6f44524b5a1a23aa54dea0ed68e4b048b5b0b929c0964aeb576d2eecdc2877a0d054cd73e8afd67ad936016a5e044ae9b95a507219a9f930a7bf4775ec
+  data.tar.gz: e8660e048b3b58049bed277b207a69a5c763eac89049c251b750e0d4e111bb003030987540758686f175f4cfd2be76959a3534262faf57b5475f7982286a966c

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright 2014 OpeNER Project Consortium
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -1,11 +1,15 @@
 Introduction
 ------------
 
-The tokenizer tokenizes a text into sentences and words. 
+The tokenizer tokenizes a text into sentences and words.
 
 ### Confused by some terminology?
 
-This software is part of a larger collection of natural language processing tools known as "the OpeNER project". You can find more information about the project at [the OpeNER portal](http://opener-project.github.io). There you can also find references to terms like KAF (an XML standard to represent linguistic annotations in texts), component, cores, scenario's and pipelines.
+This software is part of a larger collection of natural language processing
+tools known as "the OpeNER project". You can find more information about the
+project at [the OpeNER portal](http://opener-project.github.io). There you can
+also find references to terms like KAF (an XML standard to represent linguistic
+annotations in texts), component, cores, scenario's and pipelines.
 
 Quick Use Example
 -----------------
@@ -20,13 +24,14 @@ output KAF by default.
 
 ### Command line interface
 
-You should now be able to call the tokenizer as a regular shell
-command: by its name. Once installed the gem normally sits in your path so you can call it directly from anywhere.
+You should now be able to call the tokenizer as a regular shell command: by its
+name. Once installed the gem normally sits in your path so you can call it
+directly from anywhere.
 
 Tokenizing some text:
 
     echo "This is English text" | tokenizer -l en --no-kaf
-    
+
 Will result in
 
     <?xml version="1.0" encoding="UTF-8" standalone="no"?>
@@ -45,11 +50,13 @@ Will result in
       </text>
     </KAF>
 
-  The available languages for tokenization are: English (en), German (de), Dutch (nl), French (fr), Spanish (es), Italian (it)
+The available languages for tokenization are: English (en), German (de), Dutch
+(nl), French (fr), Spanish (es), Italian (it)
 
 #### KAF input format
 
-The tokenizer is capable of taking KAF as input, and actually does so by default. You can do so like this:
+The tokenizer is capable of taking KAF as input, and actually does so by
+default. You can do so like this:
 
     echo "<?xml version='1.0' encoding='UTF-8' standalone='no'?><KAF version='v1.opener' xml:lang='en'><raw>This is what I call, a test!</raw></KAF>" | tokenizer
 
@@ -72,7 +79,8 @@ Will result in
       </text>
     </KAF>
 
-If the argument -k (--kaf) is passed, then the argument -l (--language) is ignored.
+If the argument -k (--kaf) is passed, then the argument -l (--language) is
+ignored.
 
 ### Webservices
 
@@ -80,7 +88,8 @@ You can launch a language identification webservice by executing:
 
     tokenizer-server
 
-This will launch a mini webserver with the webservice. It defaults to port 9292, so you can access it at <http://localhost:9292>.
+This will launch a mini webserver with the webservice. It defaults to port 9292,
+so you can access it at <http://localhost:9292>.
 
 To launch it on a different port provide the `-p [port-number]` option like this:
 
@@ -88,19 +97,25 @@ To launch it on a different port provide the `-p [port-number]` option like this
 
 It then launches at <http://localhost:1234>
 
-Documentation on the Webservice is provided by surfing to the urls provided above. For more information on how to launch a webservice run the command with the ```-h``` option.
+Documentation on the Webservice is provided by surfing to the urls provided
+above. For more information on how to launch a webservice run the command with
+the `--help` option.
 
 
 ### Daemon
 
-Last but not least the tokenizer comes shipped with a daemon that can read jobs (and write) jobs to and from Amazon SQS queues. For more information type:
+Last but not least the tokenizer comes shipped with a daemon that can read jobs
+(and write) jobs to and from Amazon SQS queues. For more information type:
 
-    tokenizer-daemon -h
+    tokenizer-daemon --help
 
 Description of dependencies
 ---------------------------
 
-This component runs best if you run it in an environment suited for OpeNER components. You can find an installation guide and helper tools in the [OpeNER installer](https://github.com/opener-project/opener-installer) and [an installation guide on the Opener Website](http://opener-project.github.io/getting-started/how-to/local-installation.html)
+This component runs best if you run it in an environment suited for OpeNER
+components. You can find an installation guide and helper tools in the
+[OpeNER installer](https://github.com/opener-project/opener-installer) and
+[an installation guide on the Opener Website](http://opener-project.github.io/getting-started/how-to/local-installation.html).
 
 At least you need the following system setup:
 
@@ -113,16 +128,20 @@ At least you need the following system setup:
 
 * Maven (for building the Gem)
 
-
 Language Extension
 ------------------
 
-The tokenizer module is a wrapping around a Perl script, which performs the actual tokenization based on rules (when to break a character sequence). The tokenizer already supports a lot of languages. Have a look to the core script to figure out how to extend to new languages.
+The tokenizer module is a wrapping around a Perl script, which performs the
+actual tokenization based on rules (when to break a character sequence). The
+tokenizer already supports a lot of languages. Have a look to the core script to
+figure out how to extend to new languages.
 
 The Core
 --------
 
-The component is a fat wrapper around the actual language technology core. The core is a rule based tokenizer implemented in Perl. You can find the core technologies in the following repositories:
+The component is a fat wrapper around the actual language technology core. The
+core is a rule based tokenizer implemented in Perl. You can find the core
+technologies in the following repositories:
 
 * [tokenizer-base](http://github.com/opener-project/tokenizer-base)
 
@@ -135,9 +154,8 @@ Where to go from here
 Report problem/Get help
 -----------------------
 
-If you encounter problems, please email <support@opener-project.eu> or leave an issue in the 
-[issue tracker](https://github.com/opener-project/tokenizer/issues).
-
+If you encounter problems, please email <support@opener-project.eu> or leave an
+issue in the [issue tracker](https://github.com/opener-project/tokenizer/issues).
 
 Contributing
 ------------

data/bin/tokenizer-daemon

@@ -1,9 +1,10 @@
 #!/usr/bin/env ruby
-#
-require 'rubygems'
+
 require 'opener/daemons'
 
-exec_path = File.expand_path("../../exec/tokenizer.rb", __FILE__)
-Opener::Daemons::Controller.new(:name=>"tokenizer",
-                                :exec_path=>exec_path)
+controller = Opener::Daemons::Controller.new(
+  :name      => 'opener-tokenizer',
+  :exec_path => File.expand_path('../../exec/tokenizer.rb', __FILE__)
+)
 
+controller.run

data/bin/tokenizer-server

@@ -1,8 +1,10 @@
 #!/usr/bin/env ruby
 
-require 'puma/cli'
+require 'opener/webservice'
 
-rack_config = File.expand_path('../../config.ru', __FILE__)
+parser = Opener::Webservice::OptionParser.new(
+  'opener-tokenizer',
+  File.expand_path('../../config.ru', __FILE__)
+)
 
-cli = Puma::CLI.new([rack_config] + ARGV)
-cli.run
+parser.run

data/exec/tokenizer.rb

@@ -1,8 +1,9 @@
 #!/usr/bin/env ruby
-#
+
 require 'opener/daemons'
-require 'opener/tokenizer'
 
-options = Opener::Daemons::OptParser.parse!(ARGV)
-daemon = Opener::Daemons::Daemon.new(Opener::Tokenizer, options)
+require_relative '../lib/opener/tokenizer'
+
+daemon = Opener::Daemons::Daemon.new(Opener::Tokenizer)
+
 daemon.start

data/lib/opener/tokenizer.rb

@@ -2,7 +2,6 @@ require 'opener/tokenizers/base'
 require 'nokogiri'
 require 'open3'
 require 'optparse'
-require 'opener/core'
 
 require_relative 'tokenizer/version'
 require_relative 'tokenizer/cli'
@@ -41,8 +40,10 @@ module Opener
     #
     # @option options [Array] :args Collection of arbitrary arguments to pass
     #  to the individual tokenizer commands.
+    #
     # @option options [String] :language The language to use for the
     #  tokenization process.
+    #
     # @option options [TrueClass|FalseClass] :kaf When set to `true` the input
     #  is assumed to be KAF.
     #
@@ -64,19 +65,21 @@ module Opener
         else
           language = options[:language]
         end
-        
+
         unless valid_language?(language)
           raise ArgumentError, "The specified language (#{language}) is invalid"
         end
-        
+
         kernel = language_constant(language).new(:args => options[:args])
-        
-        stdout, stderr, process = Open3.capture3(*kernel.command.split(" "), :stdin_data => input)
+
+        stdout, stderr, process = Open3.capture3(
+          *kernel.command.split(" "),
+          :stdin_data => input
+        )
+
         raise stderr unless process.success?
+
         return stdout
-        
-      rescue Exception => error
-        return Opener::Core::ErrorLayer.new(input, error.message, self.class).add
       end
     end
 

data/lib/opener/tokenizer/server.rb

@@ -1,5 +1,3 @@
-require 'sinatra/base'
-require 'httpclient'
 require 'opener/webservice'
 
 module Opener
@@ -7,10 +5,11 @@ module Opener
     ##
     # Text tokenizer server powered by Sinatra.
     #
-    class Server < Webservice
+    class Server < Webservice::Server
       set :views, File.expand_path('../views', __FILE__)
-      text_processor Tokenizer
-      accepted_params :input, :kaf, :language
+
+      self.text_processor  = Tokenizer
+      self.accepted_params = [:input, :kaf, :language]
     end # Server
   end # Tokenizer
 end # Opener

data/lib/opener/tokenizer/version.rb

@@ -1,5 +1,5 @@
 module Opener
   class Tokenizer
-    VERSION = "1.1.2"
+    VERSION = '2.0.0'
   end
 end

data/opener-tokenizer.gemspec

@@ -9,6 +9,8 @@ Gem::Specification.new do |gem|
   gem.homepage    = 'http://opener-project.github.com/'
   gem.has_rdoc    = "yard"
 
+  gem.license = 'Apache 2.0'
+
   gem.required_ruby_version = '>= 1.9.2'
 
   gem.files = Dir.glob([
@@ -16,20 +18,17 @@ Gem::Specification.new do |gem|
     'lib/**/*',
     'config.ru',
     '*.gemspec',
-    'README.md'
+    'README.md',
+    'LICENSE.txt'
   ]).select { |file| File.file?(file) }
 
   gem.executables = Dir.glob('bin/*').map { |file| File.basename(file) }
 
-  gem.add_dependency 'opener-tokenizer-base', '>= 0.3.1'
-  gem.add_dependency 'opener-webservice'
-
   gem.add_dependency 'nokogiri'
-  gem.add_dependency 'sinatra', '~>1.4.2'
-  gem.add_dependency 'httpclient'
-  gem.add_dependency 'opener-daemons'
-  gem.add_dependency 'opener-core', '>= 1.0.2'
-  gem.add_dependency 'puma'
+  gem.add_dependency 'opener-tokenizer-base', '~> 1.0'
+  gem.add_dependency 'opener-webservice', '~> 2.1'
+  gem.add_dependency 'opener-daemons', '~> 2.1'
+  gem.add_dependency 'opener-core', '~> 2.0'
 
   gem.add_development_dependency 'rspec'
   gem.add_development_dependency 'cucumber'

metadata

@@ -1,185 +1,143 @@
 --- !ruby/object:Gem::Specification
 name: opener-tokenizer
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 2.0.0
 platform: ruby
 authors:
 - development@olery.com
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-19 00:00:00.000000000 Z
+date: 2014-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: opener-tokenizer-base
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 0.3.1
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 0.3.1
-  prerelease: false
-  type: :runtime
-- !ruby/object:Gem::Dependency
-  name: opener-webservice
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: nokogiri
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: opener-tokenizer-base
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
+        version: '1.0'
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: sinatra
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.2
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: opener-webservice
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.2
-  prerelease: false
+        version: '2.1'
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: httpclient
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
-  type: :runtime
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: opener-daemons
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
+        version: '2.1'
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: opener-core
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.2
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: opener-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.2
-  prerelease: false
+        version: '2.0'
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: puma
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  type: :development
   prerelease: false
-  type: :runtime
-- !ruby/object:Gem::Dependency
-  name: rspec
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: cucumber
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: cucumber
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: pry
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :development
 description: Gem that wraps up the the tokenizer cores
-email:
+email: 
 executables:
 - tokenizer
 - tokenizer-daemon
@@ -187,6 +145,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE.txt
 - README.md
 - bin/tokenizer
 - bin/tokenizer-daemon
@@ -202,26 +161,28 @@ files:
 - lib/opener/tokenizer/views/result.erb
 - opener-tokenizer.gemspec
 homepage: http://opener-project.github.com/
-licenses: []
+licenses:
+- Apache 2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.9.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
+rubyforge_project: 
 rubygems_version: 2.2.2
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Gem that wraps up the the tokenizer cores
 test_files: []
+has_rdoc: yard