proiel 1.0.0 → 1.0.1

data/lib/proiel/tokenization.rb

@@ -73,9 +73,9 @@ module PROIEL
       raise ArgumentError, 'invalid language tag' unless language_tag.is_a?(String)
       raise ArgumentError, 'invalid form' unless form.is_a?(String)
 
-      if form[/\W+/]
+      if form[/[^[:word:]]+/]
         # Split on any non-word character like a space or punctuation
-        form.split(/(\W+)/)
+        form.split(/([^[:word:]]+)/)
       elsif @@regexes.key?(language_tag) and form[@@regexes[language_tag]]
         # Apply language-specific pattern
         form.match(@@regexes[language_tag]).captures

data/lib/proiel/version.rb

@@ -1,9 +1,9 @@
 #--
-# Copyright (c) 2015 Marius L. Jøhndal
+# Copyright (c) 2015-2016 Marius L. Jøhndal
 #
 # See LICENSE in the top-level source directory for licensing terms.
 #++
 module PROIEL
   # Gem version
-  VERSION = '1.0.0'
+  VERSION = '1.0.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proiel
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Marius L. Jøhndal
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-28 00:00:00.000000000 Z
+date: 2016-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -167,10 +167,15 @@ files:
 - lib/proiel/citations.rb
 - lib/proiel/div.rb
 - lib/proiel/positional_tag.rb
+- lib/proiel/proiel_xml/proiel-1.0/proiel-1.0.md
 - lib/proiel/proiel_xml/proiel-1.0/proiel-1.0.xsd
 - lib/proiel/proiel_xml/proiel-1.0/teilite.xsd
 - lib/proiel/proiel_xml/proiel-1.0/xml.xsd
+- lib/proiel/proiel_xml/proiel-2.0/XMLSchema.xsd
+- lib/proiel/proiel_xml/proiel-2.0/make
+- lib/proiel/proiel_xml/proiel-2.0/proiel-2.0.html
 - lib/proiel/proiel_xml/proiel-2.0/proiel-2.0.xsd
+- lib/proiel/proiel_xml/proiel-2.0/xs3p.xsl
 - lib/proiel/proiel_xml/reader.rb
 - lib/proiel/proiel_xml/schema.rb
 - lib/proiel/proiel_xml/validator.rb
@@ -202,7 +207,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: A library for working with treebanks using the PROIEL dependency format