proiel 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 39313c422eb3b2d2f3ad565c0cde3cbd3ddb5271
-  data.tar.gz: 1edadad95bbaad82d4d7ab1a9cc409f8e80d3a74
+  metadata.gz: 66a5e6fc550b5d017d0b49396ec931624cc9cc12
+  data.tar.gz: a89f0936a25eb4092728ac527f8816095f721818
 SHA512:
-  metadata.gz: 41bf1b5bcb3c8d8318128ea146b2609d02942d711553876d71c29cafc948312e79e8cd2e448fef751ca25c685c3f0d57a924004a46bdb3496a8f9913772e3e48
-  data.tar.gz: add1511098c62bdd4ee59fdd53e55b4b331595a5a5e02320e97dadff194e8b0b96fb24bd48511a02933d70a83208e0f5d40093a49eec959917177bf59589cbb7
+  metadata.gz: c956426f6bd2b6e90b9254a72f256d14937c99d0ca5183897b414fd17c6a88803082c0c88690b1ad695347b29cc1bfd86de2a933cfcbff453b4fb93ad4b9e623
+  data.tar.gz: 756f1beb29e9ec3174e9b65aa7a52428a6fa87f1d9db9bec18e112afb26ff372c05d333e96e2c60b889d4a8cd72540c315ecd0719bae2ba3e2a5d6e8bbe8b4b8

data/lib/proiel/tokenization.rb

@@ -59,6 +59,8 @@ module PROIEL
       form and form.length > 1
     end
 
+    WORD_PATTERN = /([^[\u{E000}-\u{F8FF}][[:word:]]]+)/
+
     # Splits a token form using the tokenization patterns that apply for a
     # the specified language. Tokenization patterns must already have been
     # loaded.
@@ -73,9 +75,9 @@ module PROIEL
       raise ArgumentError, 'invalid language tag' unless language_tag.is_a?(String)
       raise ArgumentError, 'invalid form' unless form.is_a?(String)
 
-      if form[/[^[:word:]]+/]
+      if form[WORD_PATTERN]
         # Split on any non-word character like a space or punctuation
-        form.split(/([^[:word:]]+)/)
+        form.split(WORD_PATTERN)
       elsif @@regexes.key?(language_tag) and form[@@regexes[language_tag]]
         # Apply language-specific pattern
         form.match(@@regexes[language_tag]).captures

data/lib/proiel/version.rb

@@ -5,5 +5,5 @@
 #++
 module PROIEL
   # Gem version
-  VERSION = '1.1.0'
+  VERSION = '1.1.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proiel
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Marius L. Jøhndal
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-04 00:00:00.000000000 Z
+date: 2016-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json