tokenizer 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f536183270ebc769890d1210adfd255a4df593a1
-  data.tar.gz: 4396fc6566fe0703e326a6c135d644c071fd4f28
+  metadata.gz: 6d33919dbe66ccc9e7bb95e6239b4da8d9442f4d
+  data.tar.gz: 4eaf400b1811648d1c22ea993a981c5169af09af
 SHA512:
-  metadata.gz: e14294be0b8ac2a341bab0dea72196e76e82a8192432694dcca0a1456d86290f48ba332b56a19a0401b38360643734c99333ef2e70c7a2a9d1801b1c7618a9bf
-  data.tar.gz: d603d33571f3ae9b1a1721784ad5da0e531acd763811989ef34d91b453713b3558bea370a0ca0c13a5c8fef5a5d0ac0a9ab71a2e3be7ac6b47a328624df67d82
+  metadata.gz: 0fcd6304e8e967364756b5e2f2c02ecdec49e7563f1311fb695d29a3aa84640ec57d79db4e4a0581eec0a5363bc8e92750649ce859ad0f8304734274f896da1d
+  data.tar.gz: 2d34cf0948e0c5fab4b76730f8adcac4b7a5ee78ec90c637de2a9de1bdad234dc95fbef264ad941657c13b6ebc9aa53c8ee7485bcc06a2c47caf697fb5e3cab0

data/README.rdoc

@@ -11,7 +11,7 @@
 {<img src="https://img.shields.io/gemnasium/arbox/tokenizer.svg" alt="Dependency Status" />}[https://gemnasium.com/arbox/tokenizer]
 
 == DESCRIPTION
-A simple multilingual tokenizer -- a linguistic tool intended to split a text
+A simple multilingual tokenizer -- a linguistic tool intended to split a written text
 into tokens for NLP tasks. This tool provides a CLI and a library for
 linguistic tokenization which is an anavoidable step for many HLT (Human
 Language Technology) tasks in the preprocessing phase for further syntactic,
@@ -45,17 +45,17 @@ You can use +Tokenizer+ in two ways.
 
 * As a library for embedded tokenization:
     > require 'tokenizer'
-    > de_tokenizer = Tokenizer::Tokenizer.new
+    > de_tokenizer = Tokenizer::WhitespaceTokenizer.new
     > de_tokenizer.tokenize('Ich gehe in die Schule!')
     > => ["Ich", "gehe", "in", "die", "Schule", "!"]
 
 * Customizable PRE and POST list
     > require 'tokenizer'
-    > de_tokenizer = Tokenizer::Tokenizer.new(:de, { post: Tokenizer::Tokenizer::POST + ['|'] })
+    > de_tokenizer = Tokenizer::WhitespaceTokenizer.new(:de, { post: Tokenizer::Tokenizer::POST + ['|'] })
     > de_tokenizer.tokenize('Ich gehe|in die Schule!')
     > => ["Ich", "gehe", "|in", "die", "Schule", "!"]
 
-See documentation in the Tokenizer::Tokenizer class for details
+See documentation in the Tokenizer::WhitespaceTokenizer class for details
 on particular methods.
 
 == SUPPORT

data/lib/tokenizer/tokenizer.rb

@@ -5,7 +5,7 @@
 # A namespace for all project related stuff.
 module Tokenizer
   # Simple whitespace based tokenizer with configurable punctuation detection.
-  class Tokenizer
+  class WhitespaceTokenizer
     # Default whitespace separator.
     FS = Regexp.new('[[:blank:]]+')
 
@@ -64,10 +64,18 @@ module Tokenizer
 
     private
 
-    # @param [String] User defined string to be tokenized.
+    # @param [String] str User defined string to be tokenized.
     # @return [String] A new modified string.
     def sanitize_input(str)
       str.chomp.strip
     end
   end # class
+
+  # @deprecated Use {WhitespaceTokenizer} instead.
+  class Tokenizer < WhitespaceTokenizer
+    def initialize(*args)
+      warn '[Deprecated!] Use WhitespaceTokenizer instead.'
+      super(*args)
+    end
+  end
 end # module

data/lib/tokenizer/version.rb

@@ -1,3 +1,3 @@
 module Tokenizer
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizer
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrei Beliankou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-11 00:00:00.000000000 Z
+date: 2016-01-20 00:00:00.000000000 Z
 dependencies: []
 description: A simple multilingual tokenizer for NLP tasks. This tool provides a CLI
   and a library for linguistic tokenization which is an anavoidable step for many