string-eater 0.2.0 → 0.2.1

data/ext/string-eater/c-tokenizer.c

@@ -109,15 +109,14 @@ static VALUE tokenize_string(VALUE self,
     }
   }
 
-  ix = str_len;
   curr_token_ix = n_tokens - 1;
 
-  if(curr_token_ix == next_token_to_extract_ix)
+  if(ix < str_len && curr_token_ix == next_token_to_extract_ix)
   {
     rb_hash_aset(extracted_tokens,
         rb_ary_entry(tokens_to_extract_names, curr_token_ix),
         rb_usascii_str_new(input_string + startpoint,
-          ix - startpoint));
+          str_len - startpoint));
   }
 
   return extracted_tokens;

data/lib/version.rb

@@ -2,7 +2,7 @@ module StringEater
   module VERSION 
     MAJOR = 0
     MINOR = 2
-    PATCH = 0
+    PATCH = 1
     PRE   = nil
     STRING = [MAJOR, MINOR, PATCH, PRE].compact.join('.')
   end

data/spec/nginx_spec.rb

@@ -9,6 +9,7 @@ describe NginxLogTokenizer do
   before(:each) do
     @tokenizer = NginxLogTokenizer.new
     @str = '73.80.217.212 - - [01/Aug/2012:09:14:25 -0500] "GET /this_is_a_url HTTP/1.1" 304 152 "http://referrer.com" "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)" "-" "there could be" other "stuff here"'
+    @str2 = '73.80.217.212 - - [01/Aug/2012:09:14:25 -0500] "GET /this_is_a_url HTTP/1.1" 304 152 "http://referrer.com" "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)" "-"'
    end
 
    {
@@ -24,4 +25,8 @@ describe NginxLogTokenizer do
         end
       end
 
+      it "should handle there not being a remainder correctly" do
+        @tokenizer.tokenize!(@str2).remainder.should be_nil
+      end
+
 end

data/spec/string_eater_spec.rb

@@ -93,6 +93,12 @@ describe Example1 do
       @tokenizer.tokenize!("a b|#{s}").third_word.should == s
     end
 
+    it "should work if the last delimeter is missing and the second-to-last field is not used" do
+      s = "a b"
+#      @tokenizer.extract_all_fields
+      @tokenizer.tokenize!(s).third_word.should be_nil
+    end
+
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: string-eater
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-21 00:00:00.000000000 Z
+date: 2012-08-22 00:00:00.000000000 Z
 dependencies: []
 description: Fast string tokenizer. Nom strings.
 email:
@@ -64,4 +64,3 @@ test_files:
 - spec/nginx_spec.rb
 - spec/spec_helper.rb
 - spec/string_eater_spec.rb
-has_rdoc: