mmapscanner 0.2 → 0.3

data/README.md

@@ -43,13 +43,18 @@ Usage
 * size, length は mmap(2) したサイズを返します。
 * to_s は mmap(2) した領域を String で返します。Encoding は常に ASCII-8BIT です。
 * slice は mmap(2) した領域の一部を新たな MmapScanner オブジェクトで返します。
-* scan は正規表現に一致した部分を返し、ポインタを進めます。一致しない場合は nil を返します。
+* scan はポインタ位置で正規表現との一致を試みます。一致した部分を返し、ポインタを進めます。一致しない場合は nil を返します。
+* scan_until は scan と同じですが、現在のポインタの位置以降で一致を試みます。
 * check は scan と同じですが、ポインタを進めません。
 * skip は scan と同じですが、一致したバイト数を返します。
 * match? は check と同じですが、一致したバイト数を返します。
 * peek は指定したバイト数分のデータを返します。ポインタは進みません。
 * eos? はポインタが末尾に達していると true を返します。
 * rest はポインタ以降のデータを返します。
+* matched は正規表現に一致した部分を MmapScanner オブジェクトで返します。
+* matched(n) は正規表現の n番目の括弧に一致した部分を MmapScanner オブジェクトで返します。
+* pos は現在のポインタの位置を返します。
+* pos= でポインタ位置を変更することができます。
 
 Copyright
 ---------

data/ext/mmapscanner.c

@@ -13,6 +13,15 @@ typedef struct {
     size_t size;
 } mmap_data_t;
 
+typedef struct {
+    size_t offset;
+    size_t size;
+    size_t pos;
+    int matched;
+    size_t matched_pos;
+    struct re_registers regs;
+} mmapscanner_t;
+
 static void mmap_free(mmap_data_t *data)
 {
     if (data->ptr)
@@ -33,12 +42,31 @@ static VALUE create_mmap_object(int fd, size_t offset, size_t size)
     return Data_Wrap_Struct(cMmap, 0, mmap_free, data);
 }
 
+static void mmapscanner_free(mmapscanner_t *ms)
+{
+    onig_region_free(&ms->regs, 0);
+    free(ms);
+}
+
+VALUE allocate(VALUE klass)
+{
+    mmapscanner_t *ms;
+    ms = malloc(sizeof *ms);
+    ms->offset = 0;
+    ms->size = 0;
+    ms->pos = 0;
+    ms->matched_pos = 0;
+    onig_region_init(&ms->regs);
+    return Data_Wrap_Struct(klass, 0, mmapscanner_free, ms);
+}
+
 static VALUE initialize(int argc, VALUE *argv, VALUE obj)
 {
     VALUE src, voffset, vsize;
     size_t offset, size;
     size_t src_offset, src_size;
     VALUE src_data;
+    mmapscanner_t *ms;
 
     rb_scan_args(argc, argv, "12", &src, &voffset, &vsize);
     if (voffset != Qnil && NUM2LL(voffset) < 0)
@@ -47,8 +75,9 @@ static VALUE initialize(int argc, VALUE *argv, VALUE obj)
         rb_raise(rb_eRangeError, "length out of range: %lld", NUM2LL(vsize));
     offset = voffset == Qnil ? 0 : NUM2SIZET(voffset);
     if (rb_obj_class(src) == cMmapScanner) {
-        src_offset = NUM2SIZET(rb_iv_get(src, "offset"));
-        src_size = NUM2SIZET(rb_iv_get(src, "size"));
+        Data_Get_Struct(src, mmapscanner_t, ms);
+        src_offset = ms->offset;
+        src_size = ms->size;
         src_data = rb_iv_get(src, "data");
     } else if (TYPE(src) == T_FILE) {
         int fd;
@@ -68,27 +97,35 @@ static VALUE initialize(int argc, VALUE *argv, VALUE obj)
     } else {
         rb_raise(rb_eTypeError, "wrong argument type %s (expected File/String/MmapScanner)", rb_obj_classname(src));
     }
-    if (offset >= src_size)
-        rb_raise(rb_eRangeError, "length out of range: %zu >= %zu", offset, src_size);
+    if (offset > src_size)
+        rb_raise(rb_eRangeError, "length out of range: %zu > %zu", offset, src_size);
     size = vsize == Qnil ? src_size - offset : NUM2SIZET(vsize);
     if (size > src_size - offset)
         size = src_size - offset;
-    rb_iv_set(obj, "offset", SIZET2NUM(src_offset + offset));
-    rb_iv_set(obj, "size", SIZET2NUM(size));
+
+    Data_Get_Struct(obj, mmapscanner_t, ms);
+    ms->offset = src_offset + offset;
+    ms->size = size;
+    ms->pos = 0;
+    ms->matched = 0;
+    ms->matched_pos = 0;
     rb_iv_set(obj, "data", src_data);
-    rb_iv_set(obj, "pos", INT2NUM(0));
     return Qnil;
 }
 
 static VALUE size(VALUE obj)
 {
-    return rb_iv_get(obj, "size");
+    mmapscanner_t *ms;
+    Data_Get_Struct(obj, mmapscanner_t, ms);
+    return SIZET2NUM(ms->size);
 }
 
 static VALUE to_s(VALUE obj)
 {
-    size_t offset = NUM2SIZET(rb_iv_get(obj, "offset"));
-    size_t size = NUM2SIZET(rb_iv_get(obj, "size"));
+    mmapscanner_t *ms;
+    Data_Get_Struct(obj, mmapscanner_t, ms);
+    size_t offset = ms->offset;
+    size_t size = ms->size;
     VALUE data = rb_iv_get(obj, "data");
     mmap_data_t *mdata;
 
@@ -110,30 +147,35 @@ static VALUE inspect(VALUE obj)
 
 static VALUE pos(VALUE obj)
 {
-    return rb_iv_get(obj, "pos");
+    mmapscanner_t *ms;
+    Data_Get_Struct(obj, mmapscanner_t, ms);
+    return SIZET2NUM(ms->pos);
 }
 
 static VALUE set_pos(VALUE obj, VALUE pos)
 {
+    mmapscanner_t *ms;
+    Data_Get_Struct(obj, mmapscanner_t, ms);
     size_t p, size;
 
     if (NUM2LL(pos) < 0)
         rb_raise(rb_eRangeError, "out of range: %lld", NUM2LL(pos));
     p = NUM2SIZET(pos);
-    size = NUM2SIZET(rb_iv_get(obj, "size"));
+    size = ms->size;
     if (p > size)
         rb_raise(rb_eRangeError, "out of range: %zu > %zu", p, size);
-    rb_iv_set(obj, "pos", pos);
+    ms->pos = p;
     return pos;
 }
 
-static VALUE scan_sub(VALUE obj, VALUE re, int forward)
+static VALUE scan_sub(VALUE obj, VALUE re, int forward, int headonly, int sizeonly)
 {
+    mmapscanner_t *ms;
+    Data_Get_Struct(obj, mmapscanner_t, ms);
     regex_t *rb_reg_prepare_re(VALUE re, VALUE str);
     regex_t *reg;
     int tmpreg;
     int result;
-    struct re_registers regs;
     size_t old_pos, matched_len;
     char *ptr;
     size_t pos, size;
@@ -141,8 +183,8 @@ static VALUE scan_sub(VALUE obj, VALUE re, int forward)
     mmap_data_t *mdata;
 
     Check_Type(re, T_REGEXP);
-    pos = NUM2SIZET(rb_iv_get(obj, "pos"));
-    size = NUM2SIZET(rb_iv_get(obj, "size"));
+    pos = ms->pos;
+    size = ms->size;
     if (pos >= size)
         return Qnil;
     data = rb_iv_get(obj, "data");
@@ -152,17 +194,24 @@ static VALUE scan_sub(VALUE obj, VALUE re, int forward)
         Data_Get_Struct(data, mmap_data_t, mdata);
         ptr = mdata->ptr;
     }
-    ptr += NUM2SIZET(rb_iv_get(obj, "offset"));
+    ptr += ms->offset;
 
     reg = rb_reg_prepare_re(re, rb_str_new("", 0));
     tmpreg = reg != RREGEXP(re)->ptr;
     if (!tmpreg) RREGEXP(re)->usecnt++;
 
-    onig_region_init(&regs);
-    result = onig_match(reg, (UChar* )(ptr+pos),
-                        (UChar* )(ptr+size),
-                        (UChar* )(ptr+pos),
-                        &regs, ONIG_OPTION_NONE);
+    if (headonly) {
+        result = onig_match(reg, (UChar*)(ptr+pos),
+                            (UChar*)(ptr+size),
+                            (UChar*)(ptr+pos),
+                            &ms->regs, ONIG_OPTION_NONE);
+    } else {
+        result = onig_search(reg, (UChar*)(ptr+pos),
+                             (UChar*)(ptr+size),
+                             (UChar*)(ptr+pos),
+                             (UChar*)(ptr+size),
+                             &ms->regs, ONIG_OPTION_NONE);
+    }
     if (!tmpreg) RREGEXP(re)->usecnt--;
     if (tmpreg) {
         if (RREGEXP(re)->usecnt) {
@@ -175,65 +224,93 @@ static VALUE scan_sub(VALUE obj, VALUE re, int forward)
     if (result < 0)
         return Qnil;
     old_pos = pos;
-    matched_len = regs.end[0];
+    matched_len = ms->regs.end[0];
     if (forward) {
         pos += matched_len;
-        rb_iv_set(obj, "pos", SIZET2NUM(pos));
+        ms->pos = pos;
     }
-    return rb_funcall(cMmapScanner, rb_intern("new"), 3, obj, ULL2NUM(old_pos), ULL2NUM(matched_len));
+    ms->matched = 1;
+    ms->matched_pos = old_pos;
+
+    if (sizeonly)
+        return SIZET2NUM(matched_len);
+    return rb_funcall(cMmapScanner, rb_intern("new"), 3, obj, SIZET2NUM(old_pos), SIZET2NUM(matched_len));
 }
 
 static VALUE scan(VALUE obj, VALUE re)
 {
-    return scan_sub(obj, re, 1);
+    return scan_sub(obj, re, 1, 1, 0);
+}
+
+static VALUE scan_until(VALUE obj, VALUE re)
+{
+    return scan_sub(obj, re, 1, 0, 0);
 }
 
 static VALUE check(VALUE obj, VALUE re)
 {
-    return scan_sub(obj, re, 0);
+    return scan_sub(obj, re, 0, 1, 0);
 }
 
 static VALUE skip(VALUE obj, VALUE re)
 {
-    VALUE ret = scan_sub(obj, re, 1);
-    if (ret == Qnil)
-        return ret;
-    return rb_iv_get(ret, "size");
+    return scan_sub(obj, re, 1, 1, 1);
 }
 
 static VALUE match_p(VALUE obj, VALUE re)
 {
-    VALUE ret = scan_sub(obj, re, 0);
-    if (ret == Qnil)
-        return ret;
-    return rb_iv_get(ret, "size");
+    return scan_sub(obj, re, 0, 1, 1);
 }
 
 static VALUE peek(VALUE obj, VALUE size)
 {
+    mmapscanner_t *ms;
+    Data_Get_Struct(obj, mmapscanner_t, ms);
     size_t sz = NUM2SIZET(size);
-    size_t data_pos = NUM2SIZET(rb_iv_get(obj, "pos"));
-    size_t data_size = NUM2SIZET(rb_iv_get(obj, "size"));
-    if (sz > data_size - data_pos)
-        sz = data_size - data_pos;
-    return rb_funcall(cMmapScanner, rb_intern("new"), 3, obj, SIZET2NUM(data_pos), SIZET2NUM(sz));
+    if (sz > ms->size - ms->pos)
+        sz = ms->size - ms->pos;
+    return rb_funcall(cMmapScanner, rb_intern("new"), 3, obj, SIZET2NUM(ms->pos), SIZET2NUM(sz));
 }
 
 static VALUE eos_p(VALUE obj)
 {
-    size_t data_pos = NUM2SIZET(rb_iv_get(obj, "pos"));
-    size_t data_size = NUM2SIZET(rb_iv_get(obj, "size"));
-    return data_pos >= data_size ? Qtrue : Qfalse;
+    mmapscanner_t *ms;
+    Data_Get_Struct(obj, mmapscanner_t, ms);
+    return ms->pos >= ms->size ? Qtrue : Qfalse;
 }
 
 static VALUE rest(VALUE obj)
 {
-    return rb_funcall(cMmapScanner, rb_intern("new"), 2, obj, rb_iv_get(obj, "pos"));
+    mmapscanner_t *ms;
+    Data_Get_Struct(obj, mmapscanner_t, ms);
+    return rb_funcall(cMmapScanner, rb_intern("new"), 2, obj, SIZET2NUM(ms->pos));
+}
+
+static VALUE matched(int argc, VALUE *argv, VALUE obj)
+{
+    mmapscanner_t *ms;
+    Data_Get_Struct(obj, mmapscanner_t, ms);
+    VALUE nth;
+    int i = 0;
+    size_t pos, len;
+
+    if (rb_scan_args(argc, argv, "01", &nth) == 1)
+        i = NUM2LONG(nth);
+    if (ms->matched == 0)
+        return Qnil;
+    if (i < 0)
+        return Qnil;
+    if (i >= ms->regs.num_regs)
+        return Qnil;
+    pos = ms->matched_pos + ms->regs.beg[i];
+    len = ms->regs.end[i] - ms->regs.beg[i];
+    return rb_funcall(cMmapScanner, rb_intern("new"), 3, obj, SIZET2NUM(pos), SIZET2NUM(len));
 }
 
 void Init_mmapscanner(void)
 {
     cMmapScanner = rb_define_class("MmapScanner", rb_cObject);
+    rb_define_alloc_func(cMmapScanner, allocate);
     rb_define_method(cMmapScanner, "initialize", initialize, -1);
     rb_define_method(cMmapScanner, "size", size, 0);
     rb_define_method(cMmapScanner, "length", size, 0);
@@ -244,12 +321,14 @@ void Init_mmapscanner(void)
     rb_define_method(cMmapScanner, "pos", pos, 0);
     rb_define_method(cMmapScanner, "pos=", set_pos, 1);
     rb_define_method(cMmapScanner, "scan", scan, 1);
+    rb_define_method(cMmapScanner, "scan_until", scan_until, 1);
     rb_define_method(cMmapScanner, "check", check, 1);
     rb_define_method(cMmapScanner, "skip", skip, 1);
     rb_define_method(cMmapScanner, "match?", match_p, 1);
     rb_define_method(cMmapScanner, "peek", peek, 1);
     rb_define_method(cMmapScanner, "eos?", eos_p, 0);
     rb_define_method(cMmapScanner, "rest", rest, 0);
+    rb_define_method(cMmapScanner, "matched", matched, -1);
 
     cMmap = rb_define_class_under(cMmapScanner, "Mmap", rb_cObject);
 }

data/spec/mmapscanner_spec.rb

@@ -51,6 +51,21 @@ describe MmapScanner do
         subject.pos.should == 10
       end
     end
+    describe '#scan_until' do
+      it 'returns matched data as MmapScanner' do
+        subject.scan(/012/)
+        ret = subject.scan_until(/678/)
+        ret.class.should == MmapScanner
+        ret.to_s.should == '345678'
+      end
+      it 'returns nil if not matched' do
+        subject.scan_until(/321/).should be_nil
+      end
+      it 'forward current position' do
+        subject.scan_until(/456/)
+        subject.pos.should == 7
+      end
+    end
     describe '#check' do
       it 'returns matched data as MmapScanner' do
         ret = subject.check(/\d{10}/)
@@ -89,6 +104,30 @@ describe MmapScanner do
         subject.pos.should == 0
       end
     end
+    describe '#matched' do
+      it 'returns matched data after scan' do
+        subject.scan(/\d{6}/)
+        subject.matched.to_s.should == '012345'
+      end
+      it 'returns matched data after scan_until' do
+        subject.scan_until(/4567/)
+        subject.matched.to_s.should == '4567'
+      end
+      it 'returns nil if there is not matched data' do
+        subject.matched.should be_nil
+      end
+    end
+    describe '#matched(nth)' do
+      it 'returns nth part of matched string' do
+        subject.scan(/(..)(..)(..)/)
+        subject.matched(0).to_s.should == '012345'
+        subject.matched(1).to_s.should == '01'
+        subject.matched(2).to_s.should == '23'
+        subject.matched(3).to_s.should == '45'
+        subject.matched(4).should be_nil
+        subject.matched(-1).should be_nil
+      end
+    end
     describe '#peek' do
       it 'returns MmapScanner' do
         subject.peek(10).should be_instance_of MmapScanner
@@ -115,6 +154,10 @@ describe MmapScanner do
         ret.should be_instance_of MmapScanner
         ret.to_s.should == '789'
       end
+      it 'returns empty MmapScanner if it reached to end' do
+        subject.pos = 10000
+        subject.rest.to_s.should == ''
+      end
     end
     describe '.new with position' do
       it '#size is length of rest data' do
@@ -152,6 +195,13 @@ describe MmapScanner do
     let(:src){'0123456789'*1020}
     subject{MmapScanner.new(src, 100, 10000)}
     it_should_behave_like 'MmapScanner'
+    describe '.new with empty source' do
+      it 'returns empty MmapScanner' do
+        m = MmapScanner.new('')
+        m.size.should == 0
+        m.to_s.should be_empty
+      end
+    end
   end
 
   context 'with MmapScanner' do
@@ -163,5 +213,12 @@ describe MmapScanner do
     let(:src){MmapScanner.new(@file)}
     subject{MmapScanner.new(src, 100, 10000)}
     it_should_behave_like 'MmapScanner'
+    describe '.new with empty source' do
+      it 'returns empty MmapScanner' do
+        m = MmapScanner.new(src, 1020, 0)
+        m.size.should == 0
+        m.to_s.should be_empty
+      end
+    end
   end
 end

metadata

@@ -2,7 +2,7 @@
 name: mmapscanner
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: "0.2"
+  version: "0.3"
 platform: ruby
 authors: 
 - TOMITA Masahiro
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-02 00:00:00 +09:00
+date: 2011-03-21 00:00:00 +09:00
 default_executable: 
 dependencies: []