pragmatic_tokenizer 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4d4f4efd555a719dda2fd957aa2683a8846140f5
-  data.tar.gz: 7328a978a04c6a7754aab8d5b013d1174744bea5
+  metadata.gz: 3b2988beef12450f5a7f653ec5c2e8db5e11efce
+  data.tar.gz: 56d01b9c9ffef57ae3365f3519214318f7828a20
 SHA512:
-  metadata.gz: 7f0c65f7c717424af8fcc2e00ede126803ac5a4980e819d18020acb91b04e722f0e44e7be4067aeee0d35f165443ad1dd5534af5fe8f85af2ae63f4f83ece756
-  data.tar.gz: be8292abc8548194d660baa35947e4f5b7fbc365dad518a199f8741224bf1c864b214604b67d53e3265b7bda9f5ea561a1832f726c4dcbd04d40cbd1d8db7370
+  metadata.gz: 938592c183b4bd1f2fca41554d74e17b85f608549cfb218476c99c84829fec15e7ac6b2dbc4ce69b60a5780de90e0878203b5bfd54cdfbd71bfd0daed57d15f9
+  data.tar.gz: d767954ebcfd5003f1a58b0023307b93b22de30e08f491954511d1bcdafab4c3f23f5edb710f1d40515df98510bd63bcd9c427e4d2315f33b0d1641b726e998f

data/README.md

@@ -146,32 +146,66 @@ PragmaticTokenizer::Tokenizer.new(text, minimum_length: 6).tokenize
 #### `#urls`
 Extract only valid URL tokens
 
-*Not Yet Implemented*
+**Example Usage**
+```ruby
+text = "Go to http://www.example.com"
+
+PragmaticTokenizer::Tokenizer.new(text).urls
+# => ["http://www.example.com"]
+```
 
 <hr>
 
 #### `#emails`
 Extract only valid email tokens
 
-*Not Yet Implemented*
+**Example Usage**
+```ruby
+text = "Please email example@example.com for more info."
+
+PragmaticTokenizer::Tokenizer.new(text).emails
+# => ["example@example.com"]
+```
 
 <hr>
 
 #### `#hashtags`
 Extract only valid hashtag tokens
 
-*Not Yet Implemented*
+**Example Usage**
+```ruby
+text = "Find me all the #fun #hashtags and give me #backallofthem."
+
+PragmaticTokenizer::Tokenizer.new(text).hashtags
+# => ["#fun", "#hashtags", "#backallofthem"]
+```
 
 <hr>
 
 #### `#mentions`
 Extract only valid @ mention tokens
 
+**Example Usage**
+```ruby
+text = "Find me all the @awesome mentions."
+
+PragmaticTokenizer::Tokenizer.new(text).hashtags
+# => ["@awesome"]
+```
+
 <hr>
 
 #### `#emoticons`
 Extract only simple emoticon tokens
 
+**Example Usage**
+```ruby
+text = "Hello ;-) :) 😄"
+
+PragmaticTokenizer::Tokenizer.new(text).emoticons
+# => [";-)", ":)""]
+```
+
 <hr>
 
 #### `#emoji`
@@ -179,6 +213,14 @@ Extract only valid† emoji tokens
 
 *†matches all 1012 single-character Unicode Emoji (all except for two-character flags)*
 
+**Example Usage**
+```ruby
+text = "Return the emoji 👿😍😱🐔🌚."
+
+PragmaticTokenizer::Tokenizer.new(text).emoticons
+# => ["👿", "😍", "😱", "🐔", "🌚"]
+```
+
 ## Language Support
 
 The following lists the current level of support for different languages. Pull requests or help for any languages that are not fully supported would be greatly appreciated.  

data/lib/pragmatic_tokenizer/tokenizer.rb

@@ -44,19 +44,19 @@ module PragmaticTokenizer
     end
 
     def urls
-      []
+      text.split(' ').delete_if { |t| t !~ /(http|https|www)(\.|:)/ }.map { |t| t.chomp('.') }
     end
 
     def emails
-      []
+      text.split(' ').delete_if { |t| t !~ /\S+(＠|@)\S+/ }.map { |t| t.chomp('.') }
     end
 
     def hashtags
-      []
+      text.split(' ').delete_if { |t| t !~ /(#|＃)/ }.map { |t| t.chomp('.') }
     end
 
     def mentions
-      text.split(' ').delete_if { |t| t !~ /\A(@|＠)/ }
+      text.split(' ').delete_if { |t| t !~ /(@|＠)/ }.map { |t| t.chomp('.') }
     end
 
     def emoticons

data/lib/pragmatic_tokenizer/version.rb

@@ -1,3 +1,3 @@
 module PragmaticTokenizer
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pragmatic_tokenizer
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Kevin S. Dias