zscan 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 708c4bbc6710c4d02ed3def05702611c16cd187a
+  data.tar.gz: aaf8f91e2a98fcfb18d589d4e6bb6687aeb46ca9
+SHA512:
+  metadata.gz: a322da85ae70a1b35f33d02a9d5862250f59058c37109aad9d59b12b91dbdf66068f9740f69570d40c85290bdebf291b108e8b1daa26669e90af748bffa52d45
+  data.tar.gz: 5489bccc470ae3d9870cc7887f41bc43887159c1666a093cb4273709f58afcf83ada19dacf74fd74edad918fb2269e131bb1b3e2c6f757af7fc3ad8e92b53837

data/ext/extconf.rb

@@ -0,0 +1,3 @@
+require "mkmf"
+
+create_makefile 'zscan'

data/ext/zscan.c

@@ -0,0 +1,242 @@
+#include <ruby/ruby.h>
+#include <ruby/re.h>
+#include <ruby/encoding.h>
+
+typedef struct {
+  size_t pos;
+  size_t bytepos;
+} Pos;
+
+typedef struct {
+  size_t pos;
+  size_t bytepos;
+  VALUE s;
+  struct re_registers regs;
+  size_t stack_i;
+  size_t stack_cap;
+  Pos* stack;
+} ZScan;
+
+#define P ZScan* p = rb_check_typeddata(self, &zscan_type)
+
+static void zscan_mark(void* pp) {
+  ZScan* p = pp;
+  rb_gc_mark(p->s);
+}
+
+static void zscan_free(void* pp) {
+  ZScan* p = pp;
+  onig_region_free(&(p->regs), 0);
+  free(p->stack);
+  ruby_xfree(p);
+}
+
+extern size_t onig_region_memsize P_((const struct re_registers *regs));
+static size_t zscan_memsize(const void* pp) {
+  const ZScan* p = pp;
+  return p ? sizeof(*p) - sizeof(p->regs) + onig_region_memsize(&p->regs) : 0;
+}
+
+static const rb_data_type_t zscan_type = {
+  "ZScan",
+  {zscan_mark, zscan_free, zscan_memsize}
+};
+
+static VALUE zscan_alloc(VALUE klass) {
+  ZScan* p = ALLOC(ZScan);
+  MEMZERO(p, ZScan, 1);
+  onig_region_init(&(p->regs));
+  p->s = Qnil;
+  p->stack_cap = 5;
+  p->stack = (Pos*)malloc(sizeof(Pos) * 5);
+  return TypedData_Wrap_Struct(klass, &zscan_type, p);
+}
+
+static VALUE zscan_internal_init(VALUE self, VALUE v_s) {
+  P;
+  p->s = v_s;
+  return self;
+}
+
+static VALUE zscan_internal_string(VALUE self) {
+  P;
+  return p->s;
+}
+
+static VALUE zscan_pos(VALUE self) {
+  P;
+  return ULONG2NUM(p->pos);
+}
+
+static VALUE zscan_advance(VALUE self, VALUE v_diff) {
+  P;
+  long signed_n = p->pos + NUM2LONG(v_diff);
+  if (signed_n < 0) {
+    p->pos = 0;
+    p->bytepos = 0;
+    return self;
+  }
+  size_t n = signed_n;
+
+  // because there's no "reverse scan" API, we have a O(n) routine :(
+  if (n < p->pos) {
+    p->pos = 0;
+    p->bytepos = 0;
+  }
+
+  if (n > p->pos) {
+    rb_encoding* enc = rb_enc_get(p->s);
+    size_t byteend = RSTRING_LEN(p->s);
+    char* ptr = RSTRING_PTR(p->s);
+    for (; p->pos < n && p->bytepos < byteend;) {
+      int n = rb_enc_mbclen(ptr + p->bytepos, ptr + byteend, enc);
+      if (n) {
+        p->pos++;
+        p->bytepos += n;
+      } else {
+        break;
+      }
+    }
+  }
+  return self;
+}
+
+static VALUE zscan_bytepos(VALUE self) {
+  P;
+  return ULONG2NUM(p->bytepos);
+}
+
+static VALUE zscan_bytepos_eq(VALUE self, VALUE v_bytepos) {
+  P;
+  size_t bytepos = NUM2ULONG(v_bytepos);
+  size_t from, to;
+
+  if (bytepos > p->bytepos) {
+    from = p->bytepos;
+    to = bytepos;
+  } else if (bytepos < p->bytepos) {
+    from = bytepos;
+    to = p->bytepos;
+  } else {
+    return v_bytepos;
+  }
+
+  rb_encoding* enc = rb_enc_get(p->s);
+  char* ptr = RSTRING_PTR(p->s);
+  size_t diff = 0;
+  for (; from < to;) {
+    int n = rb_enc_mbclen(ptr + from, ptr + to, enc);
+    if (n) {
+      diff++;
+      from += n;
+    } else {
+      if (from < to) {
+        rb_raise(rb_eRuntimeError, "the given bytepos splits character");
+        return v_bytepos;
+      }
+      break;
+    }
+  }
+
+  if (bytepos > p->bytepos) {
+    p->pos += diff;
+  } else if (bytepos < p->bytepos) {
+    p->pos -= diff;
+  }
+  p->bytepos = bytepos;
+
+  return v_bytepos;
+}
+
+static VALUE zscan_eos_p(VALUE self) {
+  P;
+  return (p->bytepos == (size_t)RSTRING_LEN(p->s) ? Qtrue : Qfalse);
+}
+
+regex_t *rb_reg_prepare_re(VALUE re, VALUE str);
+static VALUE zscan_bmatch_p(VALUE self, VALUE pattern) {
+  P;
+  if (TYPE(pattern) == T_STRING) {
+    // todo
+  } else if (TYPE(pattern) == T_REGEXP) {
+    regex_t *re = rb_reg_prepare_re(pattern, p->s);
+    int tmpreg = re != RREGEXP(pattern)->ptr;
+    if (!tmpreg) RREGEXP(pattern)->usecnt++;
+
+    char* ptr = RSTRING_PTR(p->s);
+    UChar* ptr_end = (UChar*)(ptr + RSTRING_LEN(p->s));
+    UChar* ptr_match_from = (UChar*)(ptr + p->bytepos);
+    long ret = onig_match(re, (UChar*)ptr, ptr_end, ptr_match_from, &(p->regs), ONIG_OPTION_NONE);
+
+    if (!tmpreg) {
+      RREGEXP(pattern)->usecnt--;
+    }
+    if (tmpreg) {
+      if (RREGEXP(pattern)->usecnt) {
+        onig_free(re);
+      } else {
+        onig_free(RREGEXP(pattern)->ptr);
+        RREGEXP(pattern)->ptr = re;
+      }
+    }
+
+    if (ret == -2) {
+      rb_raise(rb_eRuntimeError, "regexp buffer overflow");
+    } else if (ret >= 0) {
+      return ULONG2NUM(p->regs.end[0]);
+    }
+  } else {
+    rb_raise(rb_eTypeError, "expect String or Regexp");
+  }
+
+  return Qnil;
+}
+
+static VALUE zscan_push_pos(VALUE self) {
+  P;
+  if (p->stack_i + 1 == p->stack_cap) {
+    p->stack_cap *= 2;
+    p->stack = (Pos*)realloc(p->stack, sizeof(Pos) * p->stack_cap);
+  }
+  Pos e = {p->pos, p->bytepos};
+  p->stack[++p->stack_i] = e;
+  return self;
+}
+
+static VALUE zscan_pop_pos(VALUE self) {
+  P;
+  if (p->stack_i) {
+    p->pos = p->stack[p->stack_i].pos;
+    p->bytepos = p->stack[p->stack_i].bytepos;
+    p->stack_i--;
+  } else {
+    p->pos = 0;
+    p->bytepos = 0;
+  }
+  return self;
+}
+
+static VALUE zscan_drop_top(VALUE self) {
+  P;
+  if (p->stack_i) {
+    p->stack_i--;
+  }
+  return self;
+}
+
+void Init_zscan() {
+  VALUE zscan = rb_define_class("ZScan", rb_cObject);
+  rb_define_alloc_func(zscan, zscan_alloc);
+  rb_define_method(zscan, "_internal_init", zscan_internal_init, 1);
+  rb_define_method(zscan, "_internal_string", zscan_internal_string, 0);
+
+  rb_define_method(zscan, "pos", zscan_pos, 0);
+  rb_define_method(zscan, "bytepos", zscan_bytepos, 0);
+  rb_define_method(zscan, "bytepos=", zscan_bytepos_eq, 1);
+  rb_define_method(zscan, "advance", zscan_advance, 1);
+  rb_define_method(zscan, "eos?", zscan_eos_p, 0);
+  rb_define_method(zscan, "bmatch?", zscan_bmatch_p, 1);
+  rb_define_method(zscan, "push_pos", zscan_push_pos, 0);
+  rb_define_method(zscan, "pop_pos", zscan_pop_pos, 0);
+  rb_define_method(zscan, "drop_top", zscan_drop_top, 0);
+}

data/lib/zscan.rb

@@ -0,0 +1,47 @@
+require_relative "../ext/zscan"
+
+class ZScan
+  VERSION = '0.1'
+
+  def initialize s, dup=false
+    _internal_init dup ? s.dup : s
+  end
+
+  def string
+    _internal_string.dup
+  end
+
+  def scan re_or_str
+    if sz = bmatch?(re_or_str)
+      r = _internal_string.byteslice bytepos, sz
+      self.bytepos += sz
+      r
+    end
+  end
+
+  def skip re_or_str
+    if sz = bmatch?(re_or_str)
+      self.bytepos += sz
+    end
+  end
+
+  def pos= new_pos
+    advance new_pos - pos
+  end
+
+  def rest
+    _internal_string.byteslice bytepos
+  end
+
+  private :_internal_init, :_internal_string
+end
+
+# coding: utf-8
+if __FILE__ == $PROGRAM_NAME
+  z = ZScan.new 'ab你好'
+  z.push_pos
+  z.scan /ab你/
+  p z.pos
+  p z.bytepos
+  z.pop_pos
+end

data/readme.md

@@ -0,0 +1,52 @@
+## Motivation
+
+A simple string scanner. Provides... much less methods than `StringScanner`.
+
+It supports either string or regexp as scan param.
+
+`pos` is by codepoints instead of bytes, use `bytepos` to locate byte position.
+
+It provides a position stack for you to efficiently manage scanning locations.
+
+It correctly scans anchors. The following codes demonstrate the behavior:
+
+```ruby
+require 'zscan'
+z = ZScan.new 'ab'
+z.pos = 1
+z.scan /(?<a)/ #=> ''
+z.scan /^/     #=> nil
+```
+
+While with `StringScanner`:
+
+```ruby
+require 'strscan'
+s = StringScanner.new 'ab'
+s.pos = 1
+s.scan /(?<a)/ #=> nil
+s.scan /^/     #=> ''
+```
+
+See also https://bugs.ruby-lang.org/issues/7092
+
+## Methods
+
+- `ZScan.new string, dup=false`
+- `scan regexp_or_string`
+- `skip regexp_or_string`
+- `bmatch? regexp_or_string` returns length of matched bytes or nil
+- `eos?`
+- `string` note: returns a COW dup
+- `rest`
+
+## Position management
+
+- `pos`
+- `pos= new_pos` note: complexity ~ `new_pos > pos ? new_pos - pos : new_pos`.
+- `bytepos`
+- `bytepos= new_bytepos` note: complexity ~ `abs(new_bytepos - bytepos)`.
+- `advance n` move forward `n` codepoints, if `n < 0`, move backward. Stops at beginning or end.
+- `push_pos` efficiently pushes current pos into the stack.
+- `pop_pos` efficiently sets current pos to top of the stack, and pops it.
+- `drop_top` drops top of pos stack without changing current pos.

data/zscan.gemspec

@@ -0,0 +1,16 @@
+Gem::Specification.new do |s|
+  s.name = "zscan"
+  s.version = "0.1"
+  s.author = "Zete Lui"
+  s.homepage = "https://github.com/luikore/zscan"
+  s.platform = Gem::Platform::RUBY
+  s.summary = "improved string scanner"
+  s.description = "improved string scanner"
+  s.required_ruby_version = ">=1.9.2"
+
+  s.files = %w"readme.md lib/zscan.rb ext/zscan.c ext/extconf.rb zscan.gemspec"
+  s.require_paths = ["lib"]
+  s.extensions = ["ext/extconf.rb"]
+  s.rubygems_version = '1.8.24'
+  s.has_rdoc = false
+end

metadata

@@ -0,0 +1,48 @@
+--- !ruby/object:Gem::Specification
+name: zscan
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Zete Lui
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-05 00:00:00.000000000 Z
+dependencies: []
+description: improved string scanner
+email: 
+executables: []
+extensions:
+- ext/extconf.rb
+extra_rdoc_files: []
+files:
+- readme.md
+- lib/zscan.rb
+- ext/zscan.c
+- ext/extconf.rb
+- zscan.gemspec
+homepage: https://github.com/luikore/zscan
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: improved string scanner
+test_files: []