wordtree 0.3.0 → 0.3.1

data/lib/wordtree/text_utils.rb

@@ -97,7 +97,10 @@ module WordTree
           s.unscan
         end
         # Move forward one word
-        s.scan(onegram_re)
+        if !s.scan(onegram_re)
+          # if we can't find a word, let's try to recover by scanning one char at a time
+          s.scan(/./m)
+        end
       end
     end
   end

data/lib/wordtree/version.rb

@@ -1,3 +1,3 @@
 module Wordtree
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

data/spec/wordtree/text_utils_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'wordtree/text_utils'
+require 'timeout'
 
 describe WordTree::TextUtils do
   context "#split_near" do
@@ -76,5 +77,13 @@ describe WordTree::TextUtils do
       expect{ |b| WordTree::TextUtils.each_ngram(sample_text, 3, &b) }.to \
         yield_successive_args("one word .", "word . two")
     end
+
+    it "doesn't hang on unexpected input" do
+      sample_text = "one word\n. two \n"
+      Timeout.timeout(3) do
+        expect{ |b| WordTree::TextUtils.each_ngram(sample_text, 1, &b) }.to \
+          yield_successive_args("one", "word", ".", "two")
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wordtree
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-07 00:00:00.000000000 Z
+date: 2014-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: virtus