string-eater 0.2.1 → 0.2.2

data/README.md

@@ -74,7 +74,7 @@ Then, we can tokenize the string like this:
     string = "Flinstone, Fred | 301 Cobblestone Way, Bedrock, NA, 00000" 
     tokenizer.tokenize! string
 
-    puts tokenizer.last_name # => "Flinestone" 
+    puts tokenizer.last_name # => "Flinstone" 
     puts tokenizer.city      # => "Bedrock" 
     puts tokenizer.state     # => "NA"
 

data/ext/string-eater/c-tokenizer.c

@@ -107,6 +107,12 @@ static VALUE tokenize_string(VALUE self,
       }
       looking_for = looking_for_token[looking_for_ix];
     }
+    else
+    {
+      /* reset token (or just keep looking, which is also fine) */
+      looking_for_ix = 0;
+      looking_for = looking_for_token[looking_for_ix];
+    }
   }
 
   curr_token_ix = n_tokens - 1;

data/lib/version.rb

@@ -2,7 +2,7 @@ module StringEater
   module VERSION 
     MAJOR = 0
     MINOR = 2
-    PATCH = 1
+    PATCH = 2
     PRE   = nil
     STRING = [MAJOR, MINOR, PATCH, PRE].compact.join('.')
   end

data/spec/string_eater_spec.rb

@@ -129,6 +129,29 @@ describe Example2 do
 
 end
 
+# an example where the split is more than one char
+class Example3 < TestedClass
+  look_for "foo="
+  add_field :foo_val
+  look_for "&"
+end
+
+describe Example3 do
+  before(:each) do
+    @tokenizer = Example3.new
+  end
+
+  describe "tokenize!" do
+    it "should find the token if there is only one occurrence of the characters in the separator" do
+      @tokenizer.tokenize!("abcd?foo=val&blah").foo_val.should == "val"
+    end
+
+    it "should still work if part of the separator token occurs" do
+      @tokenizer.tokenize!("abcd?foo_blah=baz&foo=bar&buh").foo_val.should == "bar"
+    end
+  end
+end
+
 # CTokenizer doesn't do combine_fields because
 #  writing out breakpoints is a significant slow-down
 if TestedClass.respond_to?(:combine_fields)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: string-eater
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-22 00:00:00.000000000 Z
+date: 2012-11-30 00:00:00.000000000 Z
 dependencies: []
 description: Fast string tokenizer. Nom strings.
 email: