mmapscanner 0.1a

data/README.md

@@ -0,0 +1,60 @@
+MmapScanner
+===========
+
+Description
+-----------
+
+文字列の代わりにファイルを mmap(2) した領域に対して StringScanner のようなことをするものです。
+
+Installation
+------------
+
+    $ cd ext
+    $ ruby ./extconf.rb
+    $ make
+    $ sudo make install
+
+Gem Installation
+----------------
+
+    $ gem install mmapscanner
+
+Features
+--------
+
+* ファイルを mmap(2) した領域から正規表現に適合した部分データを返します。
+* 返されるデータも MmapScanner オブジェクトで、最初にファイルから mmap(2) した領域を共有しています。
+* mmap(2) を使用しているので大量データでもメモリを消費しません。to_s することではじめて String オブジェクトを生成します。
+
+Usage
+-----
+
+* MmapScanner.new でファイルを mmap(2) します。mmap(2) できないファイルやパラメータを渡すとエラーになります。
+
+        # ファイル全体を mmap
+        ms = MmapScanner.new(File.open("filename"))
+        # ファイルの先頭 4096 バイト以降を mmap
+        ms = MmapScanner.new(File.open("filename"), 4096)
+        # ファイルの先頭 4096 バイト以降の 1234 バイト分を mmap
+        ms = MmapScanner.new(File.open("filename"), 4096, 1234)
+
+* size, length は mmap(2) したサイズを返します。
+* to_s は mmap(2) した領域を String で返します。Encoding は常に ASCII-8BIT です。
+* slice は mmap(2) した領域の一部を新たな MmapScanner オブジェクトで返します。
+* scan は正規表現に一致した部分を返し、ポインタを進めます。一致しない場合は nil を返します。
+* check は scan と同じですが、ポインタを進めません。
+* skip は scan と同じですが、一致したバイト数を返します。
+* match? は check と同じですが、一致したバイト数を返します。
+* peek は指定したバイト数分のデータを返します。ポインタは進みません。
+* eos? はポインタが末尾に達していると true を返します。
+* rest はポインタ以降のデータを返します。
+
+Copyright
+---------
+
+<dl>
+<dt>Author<dd>TOMITA Masahiro <tommy@tmtm.org>
+<dt>Copyrigh<dd>Copyright (c) 2011 TOMITA Masahiro
+<dt>License<dd>Ruby's
+</dl>
+

data/ext/extconf.rb

@@ -0,0 +1,2 @@
+require 'mkmf'
+create_makefile 'mmapscanner'

data/ext/mmapscanner.c

@@ -0,0 +1,254 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <ruby.h>
+#include <ruby/io.h>
+
+static VALUE cMmapScanner;
+
+typedef struct {
+    char *ptr;
+    size_t size;
+    size_t pos;
+} mmap_data_t;
+
+static void mmap_free(mmap_data_t *data)
+{
+    if (data->ptr)
+        munmap(data->ptr, data->size);
+    free(data);
+}
+
+static VALUE allocate(VALUE klass)
+{
+    VALUE obj;
+    mmap_data_t *data;
+
+    data = xmalloc(sizeof(mmap_data_t));
+    data->ptr = NULL;
+    data->size = 0;
+    data->pos = 0;
+    obj = Data_Wrap_Struct(klass, 0, mmap_free, data);
+    rb_iv_set(obj, "parent", Qnil);
+    return obj;
+}
+
+static VALUE initialize(int argc, VALUE *argv, VALUE obj)
+{
+    VALUE src, size, pos, p, pp;
+    int fd;
+    void *ptr;
+    mmap_data_t *data, *parent;
+    struct stat st;
+    size_t sz, offset;
+
+    Data_Get_Struct(obj, mmap_data_t, data);
+    if (data->ptr)
+        rb_raise(rb_eRuntimeError, "already initialized");
+    rb_scan_args(argc, argv, "12", &src, &pos, &size);
+    if (pos != Qnil && NUM2LL(pos) < 0)
+        rb_raise(rb_eRangeError, "position out of range: %lld", NUM2LL(pos));
+    if (size != Qnil && NUM2LL(size) < 0)
+        rb_raise(rb_eRangeError, "length out of range: %lld", NUM2LL(size));
+    offset = pos == Qnil ? 0 : NUM2SIZET(pos);
+    if (rb_obj_class(src) == cMmapScanner) {
+        Data_Get_Struct(src, mmap_data_t, parent);
+        if (offset >= parent->size)
+            rb_raise(rb_eRangeError, "length out of range: %zu >= %zu", offset, parent->size);
+        sz = size == Qnil ? parent->size - offset : NUM2SIZET(size);
+        ptr = parent->ptr + offset;
+        if (sz > parent->size - offset)
+            sz = parent->size-offset;
+        data->ptr = ptr;
+        data->size = sz;
+        p = src;
+        while ((pp = rb_iv_get(p, "parent")) != Qnil)
+            p = pp;
+        rb_iv_set(obj, "parent", p);
+        return;
+    }
+    Check_Type(src, T_FILE);
+    fd = RFILE(src)->fptr->fd;
+    fstat(fd, &st);
+    sz = size == Qnil ? st.st_size - offset : NUM2SIZET(size);
+    if (sz > st.st_size - offset)
+        sz = st.st_size - offset;
+    if ((ptr = mmap(NULL, sz, PROT_READ, MAP_SHARED, fd, offset)) == MAP_FAILED) {
+        rb_exc_raise(rb_funcall(rb_eSystemCallError, rb_intern("new"), 1, INT2FIX(errno)));
+    }
+    data->ptr = ptr;
+    data->size = sz;
+}
+
+static VALUE size(VALUE obj)
+{
+    mmap_data_t *data;
+
+    Data_Get_Struct(obj, mmap_data_t, data);
+    return ULL2NUM(data->size);
+}
+
+static VALUE to_s(VALUE obj)
+{
+    mmap_data_t *data;
+
+    Data_Get_Struct(obj, mmap_data_t, data);
+    return rb_str_new(data->ptr, data->size);
+}
+
+static VALUE slice(VALUE obj, VALUE pos, VALUE len)
+{
+    size_t offset;
+    size_t length;
+    mmap_data_t *data;
+
+    Data_Get_Struct(obj, mmap_data_t, data);
+    return rb_funcall(cMmapScanner, rb_intern("new"), 3, obj, pos, len);
+}
+
+static VALUE inspect(VALUE obj)
+{
+    rb_str_new2("#<MmapScanner>");
+}
+
+static VALUE pos(VALUE obj)
+{
+    mmap_data_t *data;
+
+    Data_Get_Struct(obj, mmap_data_t, data);
+    return ULL2NUM(data->pos);
+}
+
+static VALUE set_pos(VALUE obj, VALUE pos)
+{
+    mmap_data_t *data;
+    size_t p;
+
+    if (NUM2LL(pos) < 0)
+        rb_raise(rb_eRangeError, "out of range: %lld", NUM2LL(pos));
+    Data_Get_Struct(obj, mmap_data_t, data);
+    p = NUM2SIZET(pos);
+    if (p > data->size)
+        rb_raise(rb_eRangeError, "out of range: %zu > %zu", p, data->size);
+    data->pos = p;
+    return pos;
+}
+
+static VALUE scan_sub(VALUE obj, VALUE re, int forward)
+{
+    regex_t *rb_reg_prepare_re(VALUE re, VALUE str);
+    mmap_data_t *data;
+    regex_t *reg;
+    int tmpreg;
+    int result;
+    struct re_registers regs;
+    size_t old_pos, matched_len;
+
+    Check_Type(re, T_REGEXP);
+    Data_Get_Struct(obj, mmap_data_t, data);
+    if (data->pos >= data->size)
+        return Qnil;
+
+    reg = rb_reg_prepare_re(re, rb_str_new("", 0));
+    tmpreg = reg != RREGEXP(re)->ptr;
+    if (!tmpreg) RREGEXP(re)->usecnt++;
+
+    onig_region_init(&regs);
+    result = onig_match(reg, (UChar* )(data->ptr+data->pos),
+                        (UChar* )(data->ptr+data->size),
+                        (UChar* )(data->ptr+data->pos),
+                        &regs, ONIG_OPTION_NONE);
+    if (!tmpreg) RREGEXP(re)->usecnt--;
+    if (tmpreg) {
+        if (RREGEXP(re)->usecnt) {
+            onig_free(reg);
+        } else {
+            onig_free(RREGEXP(re)->ptr);
+            RREGEXP(re)->ptr = reg;
+        }
+    }
+    if (result < 0)
+        return Qnil;
+    old_pos = data->pos;
+    matched_len = regs.end[0];
+    if (forward)
+        data->pos += matched_len;
+    return rb_funcall(cMmapScanner, rb_intern("new"), 3, obj, ULL2NUM(old_pos), ULL2NUM(matched_len));
+}
+
+static VALUE scan(VALUE obj, VALUE re)
+{
+    return scan_sub(obj, re, 1);
+}
+
+static VALUE check(VALUE obj, VALUE re)
+{
+    return scan_sub(obj, re, 0);
+}
+
+static VALUE skip(VALUE obj, VALUE re)
+{
+    mmap_data_t *data;
+    VALUE ret = scan_sub(obj, re, 1);
+    if (ret == Qnil)
+        return ret;
+    Data_Get_Struct(ret, mmap_data_t, data);
+    return ULL2NUM(data->size);
+}
+
+static VALUE match_p(VALUE obj, VALUE re)
+{
+    mmap_data_t *data;
+    VALUE ret = scan_sub(obj, re, 0);
+    if (ret == Qnil)
+        return ret;
+    Data_Get_Struct(ret, mmap_data_t, data);
+    return ULL2NUM(data->size);
+}
+
+static VALUE peek(VALUE obj, VALUE size)
+{
+    size_t sz = NUM2SIZET(size);
+    mmap_data_t *data;
+    Data_Get_Struct(obj, mmap_data_t, data);
+    if (sz > data->size - data->pos)
+        sz = data->size - data->pos;
+    return rb_funcall(cMmapScanner, rb_intern("new"), 3, obj, SIZET2NUM(data->pos), SIZET2NUM(sz));
+}
+
+static VALUE eos_p(VALUE obj)
+{
+    mmap_data_t *data;
+    Data_Get_Struct(obj, mmap_data_t, data);
+    return data->pos >= data->size ? Qtrue : Qfalse;
+}
+
+static VALUE rest(VALUE obj)
+{
+    mmap_data_t *data;
+    Data_Get_Struct(obj, mmap_data_t, data);
+    return rb_funcall(cMmapScanner, rb_intern("new"), 2, obj, SIZET2NUM(data->pos));
+}
+
+void Init_mmapscanner(void)
+{
+    cMmapScanner = rb_define_class("MmapScanner", rb_cObject);
+    rb_define_alloc_func(cMmapScanner, allocate);
+    rb_define_method(cMmapScanner, "initialize", initialize, -1);
+    rb_define_method(cMmapScanner, "size", size, 0);
+    rb_define_method(cMmapScanner, "length", size, 0);
+    rb_define_method(cMmapScanner, "to_s", to_s, 0);
+    rb_define_method(cMmapScanner, "slice", slice, 2);
+//    rb_define_method(cMmapScanner, "[]", slice, 2);
+    rb_define_method(cMmapScanner, "inspect", inspect, 0);
+    rb_define_method(cMmapScanner, "pos", pos, 0);
+    rb_define_method(cMmapScanner, "pos=", set_pos, 1);
+    rb_define_method(cMmapScanner, "scan", scan, 1);
+    rb_define_method(cMmapScanner, "check", check, 1);
+    rb_define_method(cMmapScanner, "skip", skip, 1);
+    rb_define_method(cMmapScanner, "match?", match_p, 1);
+    rb_define_method(cMmapScanner, "peek", peek, 1);
+    rb_define_method(cMmapScanner, "eos?", eos_p, 0);
+    rb_define_method(cMmapScanner, "rest", rest, 0);
+}

data/spec/mmapscanner_spec.rb

@@ -0,0 +1,141 @@
+require 'tempfile'
+
+$LOAD_PATH.unshift "#{File.dirname __FILE__}/../ext"
+require 'mmapscanner'
+
+describe MmapScanner do
+  before do
+    tmpf = Tempfile.new 'mmapscanner'
+    tmpf.write '0123456789'*1000
+    @file = File.open(tmpf.path)
+  end
+  subject{MmapScanner.new(@file)}
+  it '#size returns size of file' do
+    subject.size.should == 10000
+  end
+  it '#to_s returns contents of file' do
+    subject.to_s.should == '0123456789'*1000
+  end
+  describe '#slice' do
+    it 'returns MmapScanner' do
+      subject.slice(10, 100).should be_instance_of MmapScanner
+    end
+  end
+  it '#inspect returns "#<MmapScanner>"' do
+    subject.inspect.should == '#<MmapScanner>'
+  end
+  it '#pos returns current position' do
+    subject.pos.should == 0
+    subject.scan(/.../)
+    subject.pos.should == 3
+  end
+  describe '#pos=' do
+    it 'change current position' do
+      subject.pos = 100
+      subject.pos.should == 100
+    end
+    it 'raise error when negative value' do
+      expect{subject.pos = -1}.to raise_error(RangeError, 'out of range: -1')
+    end
+    it 'raise error when over size' do
+      expect{subject.pos = 10001}.to raise_error(RangeError, 'out of range: 10001 > 10000')
+      expect{subject.pos = 20000}.to raise_error(RangeError, 'out of range: 20000 > 10000')
+    end
+  end
+  describe '#scan' do
+    it 'returns matched data as MmapScanner' do
+      ret = subject.scan(/\d{10}/)
+      ret.class.should == MmapScanner
+      ret.to_s.should == '0123456789'
+    end
+    it 'returns nil if not matched' do
+      subject.scan(/123/).should be_nil
+    end
+    it 'forward current position' do
+      subject.scan(/\d{10}/)
+      subject.pos.should == 10
+    end
+  end
+  describe '#check' do
+    it 'returns matched data as MmapScanner' do
+      ret = subject.check(/\d{10}/)
+      ret.class.should == MmapScanner
+      ret.to_s.should == '0123456789'
+    end
+    it 'returns nil if not matched' do
+      subject.check(/123/).should be_nil
+    end
+    it 'do not forward current position' do
+      ret = subject.check(/\d{10}/)
+      subject.pos.should == 0
+    end
+  end
+  describe '#skip' do
+    it 'returns length of matched data' do
+      subject.skip(/\d{10}/).should == 10
+    end
+    it 'returns nil if not matched' do
+      subject.skip(/123/).should be_nil
+    end
+    it 'forward current position' do
+      subject.skip(/\d{10}/)
+      subject.pos.should == 10
+    end
+  end
+  describe '#match?' do
+    it 'returns length of matched data' do
+      subject.match?(/\d{10}/).should == 10
+    end
+    it 'returns nil if not matched' do
+      subject.match?(/123/).should be_nil
+    end
+    it 'do not forward current position' do
+      subject.match?(/\d{10}/)
+      subject.pos.should == 0
+    end
+  end
+  describe '#peek' do
+    it 'returns MmapScanner' do
+      subject.peek(10).should be_instance_of MmapScanner
+    end
+    it 'do not forward current position' do
+      subject.peek(10)
+      subject.pos.should == 0
+    end
+  end
+  describe '#eos?' do
+    it 'returns true if eos' do
+      subject.pos = 10000
+      subject.eos?.should == true
+    end
+    it 'returns false if not eos' do
+      subject.pos = 9999
+      subject.eos?.should == false
+    end
+  end
+  describe '#rest' do
+    it 'returns rest data as MmapScanner' do
+      subject.pos = 9997
+      ret = subject.rest
+      ret.should be_instance_of MmapScanner
+      ret.to_s.should == '789'
+    end
+  end
+  describe '.new with position' do
+    it '#size is length of rest data' do
+      MmapScanner.new(@file, 4096).size.should == 10000-4096
+    end
+    it 'raise error when invalid position' do
+      expect{MmapScanner.new(@file, 4095)}.to raise_error(Errno::EINVAL)
+    end
+  end
+  describe '.new with length' do
+    subject{MmapScanner.new(@file, nil, 10)}
+    it '#size is specified size' do
+      subject.size.should == 10
+    end
+    it 'raise error when negative' do
+      expect{MmapScanner.new(@file, nil, -1)}.to raise_error(RangeError, 'length out of range: -1')
+    end
+  end
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification 
+name: mmapscanner
+version: !ruby/object:Gem::Version 
+  prerelease: 3
+  version: 0.1a
+platform: ruby
+authors: 
+- TOMITA Masahiro
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-02-05 00:00:00 +09:00
+default_executable: 
+dependencies: []
+
+description: 
+email: tommy@tmtm.org
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- README.md
+- ext/mmapscanner.c
+- spec/mmapscanner_spec.rb
+- ext/extconf.rb
+has_rdoc: true
+homepage: http://github.com/tmtm/mmapscanner
+licenses: 
+- Ruby's
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 1.3.1
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.0
+signing_key: 
+specification_version: 3
+summary: MmapScanner like StringScanner but it use mmap(2)-ed data
+test_files: 
+- spec/mmapscanner_spec.rb