zscan 1.0.1 → 1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8baa1419230fe03d1ec41cfa65202dce10d0512c
-  data.tar.gz: 0cf1335b2d7221ca4c6329dd6e8e2b5b0f3ac59d
+  metadata.gz: 998b97db8e9341f3920caa27bf11558954a777ba
+  data.tar.gz: 033986f8e4a4086985bca23c84f8acfabd2f5e29
 SHA512:
-  metadata.gz: a9ccf8455843673336365dd446f808cb52f1538a53c86cc69d102640e4a6afb40332e3c65f6f7a5a4178895a1095571e4328a501e64797a01a6af1cdc4256549
-  data.tar.gz: 8c3d109803e0e9d8e91b8787f98190fd7e400613772f57347ff2d0a99725202c8eb3444ba11efc7a606897d78cec5d9932638292c5ba2583fb697f9e7c8c2540
+  metadata.gz: a8c23d9f29b57e113a55e46fb024bce0337aa8ee5b8317744ed9ce9c13ddf35e4b1b7af9bbf25d67c6b14e6c30bb7de00c214692cb5fea05c2e0f4fa6116478b
+  data.tar.gz: 06ccbc8c793f873a630c4774b746ad8ca1cfa4c2895414871c0b435812bd902e3f75618a6134551b5ab3f35b2838e12c9072e0bb15ff8f6010a7ba2ca4e8a83a

data/{bench.rb → benchmark/vs-strscan.rb}

@@ -1,5 +1,5 @@
 require "strscan"
-require_relative "lib/zscan"
+require_relative "../lib/zscan"
 require "benchmark"
 
 s = "word\n"

data/benchmark/vs-unpack.rb

@@ -0,0 +1,21 @@
+require_relative "../lib/zscan"
+require "benchmark"
+
+spec = ZScan.binary_spec do
+  int8
+  double_le 2
+  single_be
+end
+
+arr = [1, 1.1, 1.2, 1.3]
+str = arr.pack 'cE2g'
+z = Zscan.new str.b
+
+puts 'reference nop group'
+puts Benchmark.measure{ 100000.times{ z.pos = 0 } }
+puts 'ZScan#unpack'
+puts Benchmark.measure{ 100000.times{ z.pos = 0; z.unpack 'cE2g' } }
+puts 'ZScan#scan_binary'
+puts Benchmark.measure{ 100000.times{ z.pos = 0; z.scan_binary spec } }
+puts 'String#unpack'
+puts Benchmark.measure{ 100000.times{ z.pos = 0; str.unpack 'cE2g' } }

data/ext/bspec_exec.inc

@@ -0,0 +1,156 @@
+// GENERATED WITH: rake gen
+#line 2 "ext/bspec_exec.inc"
+__attribute__((__noinline__))
+static VALUE bspec_exec(void** ip, char* s, VALUE a) {
+  static void* opcodes[] = { &&BS_RET, &&BS_INT8, &&BS_INT16, &&BS_INT16_SWAP, &&BS_INT32, &&BS_INT32_SWAP, &&BS_INT64, &&BS_INT64_SWAP, &&BS_UINT8, &&BS_UINT16, &&BS_UINT16_SWAP, &&BS_UINT32, &&BS_UINT32_SWAP, &&BS_UINT64, &&BS_UINT64_SWAP, &&BS_SINGLE, &&BS_SINGLE_SWAP, &&BS_DOUBLE, &&BS_DOUBLE_SWAP };
+  if (ip == NULL) {
+    return (VALUE)opcodes;
+  }
+  goto **(ip++);
+BS_RET:
+  return a;
+BS_INT8:
+  {
+    uint8_t r = ((uint8_t*)s)[0];
+    rb_ary_push(a, INT2FIX(CAST(r, int8_t)));
+    s += 1;
+    goto **(ip++);
+  }
+
+BS_INT16:
+  {
+    uint16_t r = ((uint16_t*)s)[0];
+    rb_ary_push(a, INT2FIX(CAST(r, int16_t)));
+    s += 2;
+    goto **(ip++);
+  }
+
+BS_INT16_SWAP:
+  {
+    uint16_t r = swap16(((uint16_t*)s)[0]);
+    rb_ary_push(a, INT2FIX(CAST(r, int16_t)));
+    s += 2;
+    goto **(ip++);
+  }
+
+BS_INT32:
+  {
+    uint32_t r = ((uint32_t*)s)[0];
+    rb_ary_push(a, INT2NUM(CAST(r, int32_t)));
+    s += 4;
+    goto **(ip++);
+  }
+
+BS_INT32_SWAP:
+  {
+    uint32_t r = swap32(((uint32_t*)s)[0]);
+    rb_ary_push(a, INT2NUM(CAST(r, int32_t)));
+    s += 4;
+    goto **(ip++);
+  }
+
+BS_INT64:
+  {
+    uint64_t r = ((uint64_t*)s)[0];
+    rb_ary_push(a, INT64toNUM(CAST(r, int64_t)));
+    s += 8;
+    goto **(ip++);
+  }
+
+BS_INT64_SWAP:
+  {
+    uint64_t r = swap64(((uint64_t*)s)[0]);
+    rb_ary_push(a, INT64toNUM(CAST(r, int64_t)));
+    s += 8;
+    goto **(ip++);
+  }
+
+BS_UINT8:
+  {
+    uint8_t r = ((uint8_t*)s)[0];
+    rb_ary_push(a, INT2FIX(CAST(r, uint8_t)));
+    s += 1;
+    goto **(ip++);
+  }
+
+BS_UINT16:
+  {
+    uint16_t r = ((uint16_t*)s)[0];
+    rb_ary_push(a, INT2FIX(CAST(r, uint16_t)));
+    s += 2;
+    goto **(ip++);
+  }
+
+BS_UINT16_SWAP:
+  {
+    uint16_t r = swap16(((uint16_t*)s)[0]);
+    rb_ary_push(a, INT2FIX(CAST(r, uint16_t)));
+    s += 2;
+    goto **(ip++);
+  }
+
+BS_UINT32:
+  {
+    uint32_t r = ((uint32_t*)s)[0];
+    rb_ary_push(a, UINT64toNUM(r));
+    s += 4;
+    goto **(ip++);
+  }
+
+BS_UINT32_SWAP:
+  {
+    uint32_t r = swap32(((uint32_t*)s)[0]);
+    rb_ary_push(a, UINT64toNUM(r));
+    s += 4;
+    goto **(ip++);
+  }
+
+BS_UINT64:
+  {
+    uint64_t r = ((uint64_t*)s)[0];
+    rb_ary_push(a, UINT64toNUM(r));
+    s += 8;
+    goto **(ip++);
+  }
+
+BS_UINT64_SWAP:
+  {
+    uint64_t r = swap64(((uint64_t*)s)[0]);
+    rb_ary_push(a, UINT64toNUM(r));
+    s += 8;
+    goto **(ip++);
+  }
+
+BS_SINGLE:
+  {
+    uint32_t r = ((uint32_t*)s)[0];
+    rb_ary_push(a, DBL2NUM((double)CAST(r, float)));
+    s += 4;
+    goto **(ip++);
+  }
+
+BS_SINGLE_SWAP:
+  {
+    uint32_t r = swap32(((uint32_t*)s)[0]);
+    rb_ary_push(a, DBL2NUM((double)CAST(r, float)));
+    s += 4;
+    goto **(ip++);
+  }
+
+BS_DOUBLE:
+  {
+    uint64_t r = ((uint64_t*)s)[0];
+    rb_ary_push(a, DBL2NUM(CAST(r, double)));
+    s += 8;
+    goto **(ip++);
+  }
+
+BS_DOUBLE_SWAP:
+  {
+    uint64_t r = swap64(((uint64_t*)s)[0]);
+    rb_ary_push(a, DBL2NUM(CAST(r, double)));
+    s += 8;
+    goto **(ip++);
+  }
+
+}

data/ext/bspec_opcode_names.inc

@@ -0,0 +1,3 @@
+// GENERATED WITH: rake gen
+const char* bspec_opcode_names[] = {"RET", "INT8", "INT16", "INT16_SWAP", "INT32", "INT32_SWAP", "INT64", "INT64_SWAP", "UINT8", "UINT16", "UINT16_SWAP", "UINT32", "UINT32_SWAP", "UINT64", "UINT64_SWAP", "SINGLE", "SINGLE_SWAP", "DOUBLE", "DOUBLE_SWAP"};
+long bspec_opcode_size = 19;

data/ext/zscan.c

@@ -1,23 +1,24 @@
 #include <ruby/ruby.h>
 #include <ruby/re.h>
 #include <ruby/encoding.h>
+#include <ctype.h>
+
+// todo infect check
 
 typedef struct {
-  size_t pos;
-  size_t bytepos;
+  long pos;
+  long bytepos;
 } Pos;
 
 typedef struct {
-  size_t pos;
-  size_t bytepos;
+  long pos;
+  long bytepos;
   VALUE s;
-  size_t stack_i;
-  size_t stack_cap;
+  long stack_i;
+  long stack_cap;
   Pos* stack;
 } ZScan;
 
-#define P ZScan* p = rb_check_typeddata(self, &zscan_type)
-
 static void zscan_mark(void* pp) {
   ZScan* p = pp;
   rb_gc_mark(p->s);
@@ -39,6 +40,8 @@ static const rb_data_type_t zscan_type = {
   {zscan_mark, zscan_free, zscan_memsize}
 };
 
+#define P ZScan* p = rb_check_typeddata(self, &zscan_type)
+
 static VALUE zscan_alloc(VALUE klass) {
   ZScan* p = ALLOC(ZScan);
   MEMZERO(p, ZScan, 1);
@@ -66,13 +69,12 @@ static VALUE zscan_pos(VALUE self) {
 
 static VALUE zscan_advance(VALUE self, VALUE v_diff) {
   P;
-  long signed_n = p->pos + NUM2LONG(v_diff);
-  if (signed_n < 0) {
+  long n = p->pos + NUM2LONG(v_diff);
+  if (n < 0) {
     p->pos = 0;
     p->bytepos = 0;
     return self;
   }
-  size_t n = signed_n;
 
   // because there's no "reverse scan" API, we have a O(n) routine :(
   if (n < p->pos) {
@@ -82,7 +84,7 @@ static VALUE zscan_advance(VALUE self, VALUE v_diff) {
 
   if (n > p->pos) {
     rb_encoding* enc = rb_enc_get(p->s);
-    size_t byteend = RSTRING_LEN(p->s);
+    long byteend = RSTRING_LEN(p->s);
     char* ptr = RSTRING_PTR(p->s);
     for (; p->pos < n && p->bytepos < byteend;) {
       int n = rb_enc_mbclen(ptr + p->bytepos, ptr + byteend, enc);
@@ -105,7 +107,7 @@ static VALUE zscan_bytepos(VALUE self) {
 static VALUE zscan_bytepos_eq(VALUE self, VALUE v_bytepos) {
   P;
   long signed_bytepos = NUM2LONG(v_bytepos);
-  size_t from, to, bytepos;
+  long from, to, bytepos;
 
   if (signed_bytepos > RSTRING_LEN(p->s)) {
     bytepos = RSTRING_LEN(p->s);
@@ -127,7 +129,7 @@ static VALUE zscan_bytepos_eq(VALUE self, VALUE v_bytepos) {
 
   rb_encoding* enc = rb_enc_get(p->s);
   char* ptr = RSTRING_PTR(p->s);
-  size_t diff = 0;
+  long diff = 0;
   for (; from < to;) {
     int n = rb_enc_mbclen(ptr + from, ptr + to, enc);
     if (n) {
@@ -154,7 +156,7 @@ static VALUE zscan_bytepos_eq(VALUE self, VALUE v_bytepos) {
 
 static VALUE zscan_eos_p(VALUE self) {
   P;
-  return (p->bytepos == (size_t)RSTRING_LEN(p->s) ? Qtrue : Qfalse);
+  return (p->bytepos == RSTRING_LEN(p->s) ? Qtrue : Qfalse);
 }
 
 regex_t *rb_reg_prepare_re(VALUE re, VALUE str);
@@ -262,10 +264,13 @@ static VALUE zscan_clear_pos_stack(VALUE self) {
   return self;
 }
 
-static VALUE zscan_try(VALUE self) {
-  if (!rb_block_given_p()) {
-    rb_raise(rb_eRuntimeError, "need a block");
+#define REQUIRE_BLOCK \
+  if (!rb_block_given_p()) {\
+    rb_raise(rb_eRuntimeError, "need a block");\
   }
+
+static VALUE zscan_try(VALUE self) {
+  REQUIRE_BLOCK;
   VALUE r;
   zscan_push(self);
   r = rb_yield(Qnil);
@@ -277,6 +282,175 @@ static VALUE zscan_try(VALUE self) {
   return r;
 }
 
+static VALUE zscan_zero_or_one(int argc, VALUE* argv, VALUE self) {
+  REQUIRE_BLOCK;
+  volatile VALUE a = Qnil;
+  volatile VALUE r;
+  rb_scan_args(argc, argv, "01", &a);
+  if (a == Qnil) {
+    a = rb_ary_new();
+  }
+  zscan_push(self);
+  r = rb_yield(Qnil);
+  if (RTEST(r)) {
+    rb_funcall(a, rb_intern("<<"), 1, r);
+    zscan_drop(self);
+  } else {
+    zscan_pop(self);
+  }
+  return a;
+}
+
+static VALUE zscan_zero_or_more(int argc, VALUE* argv, VALUE self) {
+  REQUIRE_BLOCK;
+  volatile VALUE a = Qnil;
+  volatile VALUE r;
+  long backpos;
+  P;
+  rb_scan_args(argc, argv, "01", &a);
+  if (a == Qnil) {
+    a = rb_ary_new();
+  }
+  for (;;) {
+    zscan_push(self);
+    backpos = p->bytepos;
+    r = rb_yield(Qnil);
+    if (RTEST(r) && backpos != p->bytepos) {
+      rb_funcall(a, rb_intern("<<"), 1, r);
+      zscan_drop(self);
+    } else {
+      zscan_pop(self);
+      break;
+    }
+  }
+  return a;
+}
+
+static VALUE zscan_one_or_more(int argc, VALUE* argv, VALUE self) {
+  REQUIRE_BLOCK;
+  volatile VALUE a = Qnil;
+  volatile VALUE r;
+
+  r = rb_yield(Qnil);
+  if (RTEST(r)) {
+    long backpos;
+    P;
+    rb_scan_args(argc, argv, "01", &a);
+    if (a == Qnil) {
+      a = rb_ary_new();
+    }
+
+    rb_funcall(a, rb_intern("<<"), 1, r);
+    for (;;) {
+      zscan_push(self);
+      backpos = p->bytepos;
+      r = rb_yield(Qnil);
+      if (RTEST(r) && backpos != p->bytepos) {
+        rb_funcall(a, rb_intern("<<"), 1, r);
+        zscan_drop(self);
+      } else {
+        zscan_pop(self);
+        break;
+      }
+    }
+    return a;
+  } else {
+    return Qnil;
+  }
+}
+
+VALUE zscan_scan_float(VALUE self) {
+  P;
+  if (RSTRING_LEN(p->s) == p->bytepos) {
+    return Qnil;
+  }
+
+  char* s = RSTRING_PTR(p->s) + p->bytepos;
+  if (isspace(s[0])) {
+    return Qnil;
+  }
+  char* e;
+  double d = strtod(s, &e);
+  if (e == s || e - s > RSTRING_LEN(p->s) - p->bytepos) {
+    return Qnil;
+  } else {
+    // it ok to use advance because the source is ascii compatible
+    zscan_advance(self, LONG2NUM(e - s));
+    return DBL2NUM(d);
+  }
+}
+
+static VALUE bspec_big_endian_p(VALUE self) {
+# ifdef DYNAMIC_ENDIAN
+  /* for universal binary of NEXTSTEP and MacOS X */
+  int init = 1;
+  char* p = (char*)&init;
+  return p[0] ? Qfalse : Qtrue;
+# elif defined(WORDS_BIGENDIAN)
+  return Qtrue;
+#else
+  return Qfalse;
+#endif
+}
+
+#define GCC_VERSION_SINCE(major, minor, patchlevel) \
+  (defined(__GNUC__) && !defined(__INTEL_COMPILER) && \
+   ((__GNUC__ > (major)) ||  \
+    (__GNUC__ == (major) && __GNUC_MINOR__ > (minor)) || \
+    (__GNUC__ == (major) && __GNUC_MINOR__ == (minor) && __GNUC_PATCHLEVEL__ >= (patchlevel))))
+
+#if GCC_VERSION_SINCE(4,3,0) || defined(__clang__)
+# define swap32(x) __builtin_bswap32(x)
+# define swap64(x) __builtin_bswap64(x)
+#endif
+
+#ifndef swap16
+# define swap16(x)	((uint16_t)((((x)&0xFF)<<8) | (((x)>>8)&0xFF)))
+#endif
+
+#ifndef swap32
+# define swap32(x)	((uint32_t)((((x)&0xFF)<<24)	\
+		|(((x)>>24)&0xFF)	\
+		|(((x)&0x0000FF00)<<8)	\
+		|(((x)&0x00FF0000)>>8)	))
+#endif
+
+#ifndef swap64
+# ifdef HAVE_INT64_T
+#  define byte_in_64bit(n) ((uint64_t)0xff << (n))
+#  define swap64(x)       ((uint64_t)((((x)&byte_in_64bit(0))<<56) 	\
+	   |(((x)>>56)&0xFF)	                \
+	   |(((x)&byte_in_64bit(8))<<40)	\
+	   |(((x)&byte_in_64bit(48))>>40)	\
+	   |(((x)&byte_in_64bit(16))<<24)	\
+	   |(((x)&byte_in_64bit(40))>>24)	\
+	   |(((x)&byte_in_64bit(24))<<8)	\
+	   |(((x)&byte_in_64bit(32))>>8)))
+# endif
+#endif
+
+// NOTE can not use sizeof in preprocessor
+#define INT64toNUM(x) (sizeof(long) == 8 ? LONG2NUM(x) : LL2NUM(x))
+#define UINT64toNUM(x) (sizeof(long) == 8 ? ULONG2NUM(x) : ULL2NUM(x))
+
+#define CAST(var, ty) *((ty*)(&(var)))
+
+#include "bspec_exec.inc"
+
+static VALUE zscan_scan_binary(VALUE self, VALUE spec) {
+  P;
+  long s_size = NUM2LONG(rb_iv_get(spec, "@s_size"));
+  if (p->bytepos + s_size > RSTRING_LEN(p->s)) {
+    return Qnil;
+  }
+  VALUE code = rb_iv_get(spec, "@code");
+  long a_size = RSTRING_LEN(code) / sizeof(void*);
+  volatile VALUE a = rb_ary_new2(a_size);
+  bspec_exec((void**)RSTRING_PTR(code), RSTRING_PTR(p->s) + p->bytepos, a);
+  zscan_bytepos_eq(self, LONG2NUM(p->bytepos + s_size));
+  return a;
+}
+
 void Init_zscan() {
   VALUE zscan = rb_define_class("ZScan", rb_cObject);
   rb_define_alloc_func(zscan, zscan_alloc);
@@ -295,5 +469,23 @@ void Init_zscan() {
   rb_define_method(zscan, "drop", zscan_drop, 0);
   rb_define_method(zscan, "restore", zscan_restore, 0);
   rb_define_method(zscan, "clear_pos_stack", zscan_clear_pos_stack, 0);
+
   rb_define_method(zscan, "try", zscan_try, 0);
+  rb_define_method(zscan, "zero_or_one", zscan_zero_or_one, -1);
+  rb_define_method(zscan, "zero_or_more", zscan_zero_or_more, -1);
+  rb_define_method(zscan, "one_or_more", zscan_one_or_more, -1);
+
+  rb_define_method(zscan, "scan_float", zscan_scan_float, 0);
+  rb_define_method(zscan, "scan_binary", zscan_scan_binary, 1);
+
+  VALUE bs = rb_define_class_under(zscan, "BinarySpec", rb_cObject);
+  rb_define_singleton_method(bs, "big_endian?", bspec_big_endian_p, 0);
+
+# include "bspec_opcode_names.inc"
+  void** opcodes = (void**)bspec_exec(NULL, NULL, Qnil);
+  for (long i = 0; i < bspec_opcode_size; i++) {
+    VALUE bytecode = rb_str_new((char*)&opcodes[i], sizeof(void*));
+    OBJ_FREEZE(bytecode);
+    rb_define_const(bs, bspec_opcode_names[i], bytecode);
+  }
 }

data/lib/zscan.rb

@@ -1,10 +1,17 @@
 require_relative "../ext/zscan"
+require_relative "zscan/instructions"
+require "date"
 
 class ZScan
-  VERSION = '1.0.1'
+  VERSION = '1.1'
 
   def initialize s, dup=false
-    _internal_init dup ? s.dup : s
+    if s.encoding.ascii_compatible?
+      s = dup ? s.dup : s
+    else
+      s = s.encode 'utf-8'
+    end
+    _internal_init s
   end
 
   def string
@@ -17,6 +24,76 @@ class ZScan
     end
   end
 
+  def scan_int radix=nil
+    negative = false
+    r = try do
+      negative = (scan(/[+\-]/) == '-')
+      if radix.nil?
+        radix =
+          if scan(/0b/i)
+            2
+          elsif scan(/0x/i)
+            16
+          elsif scan('0')
+            8
+          else
+            10
+          end
+      end
+      scan \
+        case radix
+        when 2;  /[01]+/
+        when 8;  /[0-7]+/
+        when 10; /\d+/
+        when 16; /\h+/i
+        else
+          if radix < 10
+            /[0-#{radix}]+/
+          elsif radix > 36
+            raise ArgumentError, "invalid radix #{radix}"
+          else
+            end_char = ('a'.ord + (radix - 11)).chr
+            /[\da-#{end_char}]+/i
+          end
+        end
+    end
+    if r
+      r = r.to_i radix
+      negative ? -r : r
+    end
+  end
+
+  def scan_date format, start=Date::ITALY
+    s = rest
+    d = DateTime._strptime s, format
+    if d
+      # XXX need 2 parses because the handling is very complex ...
+      dt = DateTime.strptime s, format, start rescue return nil
+
+      len = s.bytesize
+      if leftover = d[:leftover]
+        len -= leftover.bytesize
+      end
+      self.bytepos += len
+
+      dt
+    end
+  end
+
+  def unpack format
+    if format.index('@')
+      raise ArgumentError, 'position instruction @ not supported'
+    end
+    r = rest.unpack format
+    if r.index(nil)
+      return
+    end
+    # XXX pack to get parsed length because no related API is exposed ...
+    len = r.pack(format).bytesize
+    self.bytepos += len
+    r
+  end
+
   def pos= new_pos
     advance new_pos - pos
   end
@@ -57,6 +134,26 @@ class ZScan
     _internal_string.bytesize
   end
 
+  def line_index
+    _internal_string.byteslice(0, bytepos).count "\n"
+  end
+
+  def self.binary_spec &p
+    bs = BinarySpec.new
+    bs.instance_eval &p
+    bs.instance_variable_get(:@code) << BinarySpec::RET
+    bs
+  end
+
+  class BinarySpec
+    BLANK = ''.force_encoding 'binary'
+
+    def initialize
+      @code = BLANK.dup
+      @s_size = 0
+    end
+  end
+
   private :_internal_init, :_internal_string
 end
 

data/lib/zscan/instructions.rb

@@ -0,0 +1,165 @@
+# GENERATED WITH: rake gen
+class ZScan::BinarySpec
+  def int8 n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << INT8
+      @s_size += 1
+    end
+  end
+  def int16 n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << INT16
+      @s_size += 2
+    end
+  end
+  def int16_swap n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << INT16_SWAP
+      @s_size += 2
+    end
+  end
+  def int32 n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << INT32
+      @s_size += 4
+    end
+  end
+  def int32_swap n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << INT32_SWAP
+      @s_size += 4
+    end
+  end
+  def int64 n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << INT64
+      @s_size += 8
+    end
+  end
+  def int64_swap n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << INT64_SWAP
+      @s_size += 8
+    end
+  end
+  def uint8 n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << UINT8
+      @s_size += 1
+    end
+  end
+  def uint16 n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << UINT16
+      @s_size += 2
+    end
+  end
+  def uint16_swap n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << UINT16_SWAP
+      @s_size += 2
+    end
+  end
+  def uint32 n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << UINT32
+      @s_size += 4
+    end
+  end
+  def uint32_swap n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << UINT32_SWAP
+      @s_size += 4
+    end
+  end
+  def uint64 n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << UINT64
+      @s_size += 8
+    end
+  end
+  def uint64_swap n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << UINT64_SWAP
+      @s_size += 8
+    end
+  end
+  def single n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << SINGLE
+      @s_size += 4
+    end
+  end
+  def single_swap n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << SINGLE_SWAP
+      @s_size += 4
+    end
+  end
+  def double n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << DOUBLE
+      @s_size += 8
+    end
+  end
+  def double_swap n=1
+    raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
+    n.times do
+      @code << DOUBLE_SWAP
+      @s_size += 8
+    end
+  end
+  if ZScan::BinarySpec.big_endian?
+    alias int16_be int16
+    alias int16_le int16_swap
+    alias int32_be int32
+    alias int32_le int32_swap
+    alias int64_be int64
+    alias int64_le int64_swap
+    alias uint16_be uint16
+    alias uint16_le uint16_swap
+    alias uint32_be uint32
+    alias uint32_le uint32_swap
+    alias uint64_be uint64
+    alias uint64_le uint64_swap
+    alias single_be single
+    alias single_le single_swap
+    alias double_be double
+    alias double_le double_swap
+  else
+    alias int16_le int16
+    alias int16_be int16_swap
+    alias int32_le int32
+    alias int32_be int32_swap
+    alias int64_le int64
+    alias int64_be int64_swap
+    alias uint16_le uint16
+    alias uint16_be uint16_swap
+    alias uint32_le uint32
+    alias uint32_be uint32_swap
+    alias uint64_le uint64
+    alias uint64_be uint64_swap
+    alias single_le single
+    alias single_be single_swap
+    alias double_le double
+    alias double_be double_swap
+  end
+  undef int16_swap, int32_swap, int64_swap, uint16_swap, uint32_swap, uint64_swap, single_swap, double_swap
+end

data/rakefile

@@ -4,6 +4,60 @@ version     = `command grep 'VERSION =' lib/zscan.rb`[version_re]
 gem_files   = Dir.glob('{rakefile,zscan.gemspec,readme.md,**/*.{rb,c}}')
 gem_package = "zscan-#{version}.gem"
 
+bspec_types = %w[INT8 INT16 INT32 INT64 UINT8 UINT16 UINT32 UINT64 SINGLE DOUBLE]
+bspec_insns = bspec_types.flat_map{|ty|
+  if ty =~ /INT8/
+    ty
+  else
+    [ty, "#{ty}_SWAP"]
+  end
+}
+def bspec_incr ins
+  case ins
+  when /INT(\d+)/; $1.to_i / 8
+  when /SINGLE/; 4
+  when /DOUBLE/; 8
+  else; raise 'bad'
+  end
+end
+def bspec_c_type ins
+  case ins
+  when /(U?INT\d+)/; "#{$1.downcase}_t"
+  when /SINGLE/; 'float'
+  when /DOUBLE/; 'double'
+  else; raise 'bad'
+  end
+end
+def bspec_extract ins
+  type = bspec_c_type ins
+  len = bspec_incr(ins) * 8
+  r = "((uint#{len}_t*)s)[0]"
+  if ins.end_with?('SWAP')
+    r = "swap#{len}(#{r})"
+  end
+  "uint#{len}_t r = #{r}"
+end
+def bspec_convert ins
+  case ins
+  when /(U)?INT64|UINT32/
+    if ins.start_with?('U')
+      "UINT64toNUM(r)"
+    else
+      "INT64toNUM(CAST(r, int64_t))"
+    end
+  when /INT32/
+    "INT2NUM(CAST(r, int32_t))"
+  when /INT(16|8)/
+    "INT2FIX(CAST(r, #{bspec_c_type ins}))"
+  when /SINGLE/
+    "DBL2NUM((double)CAST(r, float))"
+  when /DOUBLE/
+    "DBL2NUM(CAST(r, double))"
+  else
+    raise 'bad'
+  end
+end
+
 desc "build and test"
 task :default => [:test, gem_package]
 
@@ -30,3 +84,86 @@ file gem_package => gem_files do
   end
   sh "gem build zscan.gemspec"
 end
+
+desc "generate files"
+task :gen => %w[ext/bspec_exec.inc ext/bspec_opcode_names.inc lib/zscan/instructions.rb]
+
+file 'ext/bspec_exec.inc' => __FILE__ do
+  puts "generating ext/bspec_exec.inc"
+  opcode_list = bspec_insns.map do |ins|
+    "&&BS_#{ins}"
+  end.join ', '
+
+  opcode_segs = bspec_insns.map do |ins|
+%Q{BS_#{ins}:
+  {
+    #{bspec_extract ins};
+    rb_ary_push(a, #{bspec_convert ins});
+    s += #{bspec_incr ins};
+    goto **(ip++);
+  }
+}
+  end.join "\n"
+
+  File.open 'ext/bspec_exec.inc', 'w' do |f|
+    f.puts %Q|// GENERATED WITH: rake gen
+#line 2 "ext/bspec_exec.inc"
+__attribute__((__noinline__))
+static VALUE bspec_exec(void** ip, char* s, VALUE a) {
+  static void* opcodes[] = { &&BS_RET, #{opcode_list} };
+  if (ip == NULL) {
+    return (VALUE)opcodes;
+  }
+  goto **(ip++);
+BS_RET:
+  return a;
+#{opcode_segs}
+}|
+  end
+end
+
+file 'ext/bspec_opcode_names.inc' => __FILE__ do
+  puts 'generating ext/bspec_opcode_names.inc'
+  opcode_names = bspec_insns.map(&:inspect).join ', '
+  File.open 'ext/bspec_opcode_names.inc', 'w' do |f|
+    f.puts "// GENERATED WITH: rake gen"
+    f.puts %Q|const char* bspec_opcode_names[] = {"RET", #{opcode_names}};|
+    f.puts %Q|long bspec_opcode_size = #{bspec_insns.size + 1};|
+  end
+end
+
+file 'lib/zscan/instructions.rb' => __FILE__ do
+  puts 'generating lib/zscan/instructions.rb'
+  File.open 'lib/zscan/instructions.rb', 'w' do |f|
+    f.puts "# GENERATED WITH: rake gen"
+    f.puts "class ZScan::BinarySpec"
+
+    bspec_insns.each do |ins|
+      f.puts <<-RUBY
+  def #{ins.downcase} n=1
+    raise ArgumentError, "repeat count should be >= 1, but got \#{n}" if n < 1
+    n.times do
+      @code << #{ins}
+      @s_size += #{bspec_incr ins}
+    end
+  end
+      RUBY
+    end
+
+    alias_ins = (bspec_types - ['INT8', 'UINT8']).map &:downcase
+    f.puts "  if ZScan::BinarySpec.big_endian?"
+    alias_ins.each do |ins|
+    f.puts "    alias #{ins}_be #{ins}"
+    f.puts "    alias #{ins}_le #{ins}_swap"
+    end
+    f.puts "  else"
+    alias_ins.each do |ins|
+    f.puts "    alias #{ins}_le #{ins}"
+    f.puts "    alias #{ins}_be #{ins}_swap"
+    end
+    f.puts "  end"
+    swap_ins = alias_ins.map{|ins| "#{ins}_swap"}
+    f.puts "  undef #{swap_ins.join ', '}"
+    f.puts "end"
+  end
+end

data/readme.md

@@ -4,6 +4,7 @@
 - `ZScan#pos` is the codepoint position, and `ZScan#bytepos` is byte position.
 - Correctly scans anchors and look behind predicates.
 - Pos stack manipulation.
+- Typed scanning methods: `#scan_float`, `#scan_int radix=nil`, `#scan_date format`, `#scan_binary format`.
 
 ## Install
 
@@ -22,7 +23,7 @@ z.scan /\w+/   #=> 'world'
 z.eos?         #=> true
 ```
 
-## Motivation
+## Motivation - `StringScanner`
 
 Ruby's stdlib `StringScanner` treats the scanning position as beginning of string:
 
@@ -46,41 +47,112 @@ z.scan /^/     #=> nil
 
 See also https://bugs.ruby-lang.org/issues/7092
 
+## Other motivations - `scanf` / `strptime` / `unpack`
+
+- For scan and convert, ruby's stdlib `Scanf` is slow (creates regexp array everytime called) and not possible to corporate with scanner.
+- For date parsing, `strptime` doesn't tell the parsed length.
+- For binary parsing, `unpack` is an slow interpreter, and the instructions are quite irregular.
+
 ## Essential methods
 
 - `ZScan.new string, dup=false`
 - `#scan regexp_or_string`
 - `#skip regexp_or_string`
 - `#match_bytesize regexp_or_string` return length of matched bytes or `nil`.
+- `#scan_float` scan a float number which is not starting with space. It deals with multibyte encodings for you.
+- `#scan_int radix=nil` if radix is nil, decide base by prefix: `0x` is 16, `0` is 8, `0b` is 2, otherwise 10. `radix` should be in range `2..36`.
+- `#scan_date format_string, start=Date::ITALY` scan a `DateTime` object, see also [strptime](http://rubydoc.info/stdlib/date/DateTime.strptime).
+- `#scan_binary binary_spec` optimized and readable binary scan, see below for how to create a `ZScan::BinarySpec`.
+- `#unpack format_string`
 - `#eos?`
 - `#string` note: return a dup. Don't worry the performance because it is a copy-on-write string.
 - `#rest`
 
+## String delegates
+
+For convienience
+
+- `#<< append_string`
+- `#[]= range, replace_string` note: if `range` starts before pos, moves pos left, also clears the stack.
+- `#size`
+- `#bytesize`
+
+## Parsing combinators
+
+Combinators that manage scanner pos and stack state for you. In the combinators, if the returned value of the given block is `nil` or `false`, stops iteration. Can be nested, useful for building parsers.
+
+- `#try &block` returns `block`'s return.
+- `#zero_or_one result=[], &block` try to execute 0 or 1 time, returns `result`.
+- `#zero_or_more result=[], &block` try to execute 0 or more times, also stops iteration if scanner no advance, returns `result`.
+- `#one_or_more result=[], &block` try to execute 1 or more times, also stops iteration if scanner no advance, returns `nil` or `result`.
+
 ## Pos management
 
 - `#pos`
 - `#pos= new_pos` note: complexity ~ `new_pos > pos ? new_pos - pos : new_pos`.
 - `#bytepos`
 - `#bytepos= new_bytepos` note: complexity ~ `abs(new_bytepos - bytepos)`.
+- `#line_index` line index of current position, start from `0`.
 - `#advance n` move forward `n` codepoints, if `n < 0`, move backward. Stops at beginning or end.
 - `#reset` go to beginning.
 - `#terminate` go to end of string.
 
-## Efficient pos stack manipulation
+## (Low level) Efficient pos stack manipulation
 
 - `#push` push current pos into the stack.
 - `#pop` set current pos to top of the stack, and pop it.
 - `#drop` drop top of pos stack without changing current pos.
 - `#restore` set current pos to top of the stack.
 - `#clear_pos_stack` clear pos stack.
-- `#try` try to do several scans in the given block, fall back to init pos if block returns `nil` or `false`. Returns block's return, can be nested.
 
-## String delegates
+## `ZScan::BinarySpec`
 
-- `#<< append_string`
-- `#[]= range, replace_string` note: if `range` starts before pos, moves pos left, also clears the stack.
-- `#size`
-- `#bytesize`
+Specify a sequence of binary data. Designed for binary protocol parsing. Example:
+
+```ruby
+# create a ZScan::BinarySpec
+s = ZScan.binary_spec do
+  int8        # once
+  uint32_le 2 # little endian, twice
+  double_be 1 # big endian, once
+end
+z = ZScan.new [-1, 2, 3, 4.0].pack('cI<2G') + "rest"
+z.scan_binary s #=> [-1, 2, 3, 4.0]
+z.rest #=> 'rest
+```
+
+Integer instructions:
+
+```ruby
+int8  uint8
+int16 uint16 int16_le uint16_le int16_be uint16_be
+int32 uint32 int32_le uint32_le int32_be uint32_be
+int64 uint64 int64_le uint64_le int64_be uint64_be
+```
+
+Single precision float instructions:
+
+```ruby
+single single_le single_be
+```
+
+Double precision float instructions:
+
+```ruby
+double double_le double_be
+```
+
+Endians:
+
+- (without endian suffix) native endian
+- `*_le` little endian (VAX, x86, Windows string code unit)
+- `*_be` big endian, network endian (SPARC, Java string code unit)
+
+Repeat count must be integer `>= 1`, default is `1`.
+
+It is implemented as a direct-threaded bytecode interpreter. Performance vs `String#unpack`:
+
+todo
 
 ## License
 

data/spec/binary_scan_spec.rb

@@ -0,0 +1,28 @@
+require_relative "spec_helper"
+
+describe 'ZScan binary scanning methods' do
+  it "#unpack" do
+    z = ZScan.new "\x01\x02\x03"
+    assert_raise ArgumentError do
+      z.unpack '@1C'
+    end
+    assert_equal [1, 2], (z.unpack 'CC')
+    assert_equal 2, z.pos
+    assert_equal nil, (z.unpack 'I')
+    assert_equal 2, z.pos
+  end
+  
+  it "#scan_binary" do
+    s = ZScan.binary_spec do
+      int8        # once
+      uint32_le 2 # little endian, twice
+      double_be 1 # big endian, once
+      single 1
+    end
+    a = [-1, 2, 3, 4.0, 3.0]
+    z = ZScan.new(a.pack('cI<2Gf') + 'rest')
+    b = z.scan_binary s
+    assert_equal 'rest', z.rest
+    assert_equal a, b
+  end
+end

data/spec/combinator_spec.rb

@@ -0,0 +1,52 @@
+require_relative "spec_helper"
+
+describe 'ZScan combinators' do
+  it "#try restores pos" do
+    z = ZScan.new "hello"
+    return1 = z.try do
+      z.scan 'h'
+      z.scan 'e'
+    end
+    assert_equal 'e', return1
+    assert_equal 2, z.pos
+
+    return2 = z.try do
+      z.scan 'l'
+      z.scan 'l'
+      z.scan 'p' # fails
+    end
+    assert_equal nil, return2
+    assert_equal 2, z.pos
+  end
+
+  it "#zero_or_one" do
+    z = Zscan.new "aab"
+    assert_equal ['a'], z.zero_or_one{z.scan 'a'}
+    assert_equal 1, z.pos
+
+    z = Zscan.new 'aab'
+    assert_equal [], z.zero_or_one{z.scan 'b'}
+    assert_equal 0, z.pos
+  end
+
+  it "#zero_or_more" do
+    z = Zscan.new "aab"
+    assert_equal ['a', 'a'], z.zero_or_more{z.scan 'a'}
+    assert_equal 2, z.pos
+
+    assert_equal 'aab', z.zero_or_more('aa'){z.scan 'c'; z.scan 'b'}
+
+    z = Zscan.new 'aab'
+    assert_equal [], z.zero_or_more{z.scan 'b'}
+    assert_equal 0, z.pos
+  end
+
+  it "#one_or_more" do
+    z = Zscan.new 'aab'
+    assert_equal ['a', 'a'], z.one_or_more{z.scan 'a'}
+    assert_equal 2, z.pos
+
+    z = Zscan.new 'aab'
+    assert_equal nil, z.one_or_more([]){z.scan 'b'}
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require_relative "../lib/zscan"
+require 'rspec/autorun'
+RSpec.configure do |config|
+  config.expect_with :stdlib
+  config.before :all do
+    # GC.stress = true
+  end
+end

data/spec/typed_scan_spec.rb

@@ -0,0 +1,48 @@
+require_relative "spec_helper"
+
+describe "typed scan" do
+  it "#scan_int" do
+    z = Zscan.new " 1 0b10F5 10 030"
+    assert_equal nil, z.scan_int
+    z.advance 1
+    assert_equal 1, z.scan_int(10)
+
+    z.advance 1
+    assert_equal 0b10, z.scan_int
+    assert_equal 0xF5, z.scan_int(16)
+
+    z.advance 1
+    assert_equal 12, z.scan_int(12)
+
+    z.advance 1
+    assert_equal 030, z.scan_int
+  end
+
+  it "#scan_float" do
+    z = Zscan.new " -3.5e23"
+    assert_equal nil, z.scan_float
+    z.advance 1
+    assert_equal -3.5e23, z.scan_float
+  end
+
+  it "won't overflow in #scan_float" do
+    s = '1.23E15'.byteslice 0, 4
+    z = Zscan.new s
+    assert_equal 1.23, z.scan_float
+    assert_equal 4, z.pos
+  end
+
+  it "#scan_date" do
+    z = Zscan.new " 2001 04 6 04 05 06 +7 231rest"
+    assert_equal nil, z.scan_date('%Y %U %w %H %M %S %z %N')
+    z.advance 1
+
+    d = z.scan_date '%Y %U %w %H %M %S %z %N'
+    assert_equal 0.231, d.sec_fraction
+    assert_equal 'rest', z.rest
+
+    z.pos = 1
+    z.scan_date '%Y %U %w ahoy %H %M %S %z' # bad format
+    assert_equal 1, z.pos
+  end
+end

data/spec/zscan_spec.rb

@@ -1,18 +1,10 @@
-require_relative "../lib/zscan"
-require 'rspec/autorun'
-RSpec.configure do |config|
-  config.expect_with :stdlib
-end
+require_relative "spec_helper"
 
 describe ZScan do
   before :each do
     @z = ZScan.new 'ab你好'
   end
 
-  before :all do
-    GC.stress = true
-  end
-
   it "random workflow" do
     assert_equal 2, @z.match_bytesize('ab')
     @z.pos = 4
@@ -76,21 +68,23 @@ describe ZScan do
     assert_equal 3, @z.pos
   end
 
-  it "#try restores pos" do
-    z = ZScan.new "hello"
-    return1 = z.try do
-      z.scan 'h'
-      z.scan 'e'
-    end
-    assert_equal 'e', return1
-    assert_equal 2, z.pos
+  it "#reset, #terminate and #line_index" do
+    z = ZScan.new ''
+    assert_equal 0, z.line_index
+    z.terminate
+    assert_equal 0, z.line_index
+    z.reset
+    assert_equal 0, z.line_index
 
-    return2 = z.try do
-      z.scan 'l'
-      z.scan 'l'
-      z.scan 'p' # fails
-    end
-    assert_equal nil, return2
-    assert_equal 2, z.pos
+    z = ZScan.new "a\nb\nc"
+    assert_equal 0, z.line_index
+    z.terminate
+    assert_equal 2, z.line_index
+    z.reset
+    assert_equal 0, z.line_index
+    z.pos = 1
+    assert_equal 0, z.line_index
+    z.pos = 2
+    assert_equal 1, z.line_index
   end
 end

data/zscan.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = "zscan"
-  s.version = "1.0.1" # version mapped from zscan.rb, don't change here
+  s.version = "1.1" # version mapped from zscan.rb, don't change here
   s.author = "Zete Lui"
   s.homepage = "https://github.com/luikore/zscan"
   s.platform = Gem::Platform::RUBY
@@ -9,7 +9,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = ">=1.9.2"
   s.licenses = ['BSD']
 
-  s.files = Dir.glob('{rakefile,zscan.gemspec,readme.md,**/*.{rb,c}}')
+  s.files = Dir.glob('{rakefile,zscan.gemspec,readme.md,**/*.{rb,c,inc}}')
   s.require_paths = ["lib"]
   s.extensions = ["ext/extconf.rb"]
   s.rubygems_version = '1.8.24'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zscan
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: '1.1'
 platform: ruby
 authors:
 - Zete Lui
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-07 00:00:00.000000000 Z
+date: 2013-05-14 00:00:00.000000000 Z
 dependencies: []
 description: improved string scanner, respects anchors and lookbehinds, supports codepoint
   positioning
@@ -21,11 +21,19 @@ files:
 - rakefile
 - zscan.gemspec
 - readme.md
-- bench.rb
+- benchmark/vs-strscan.rb
+- benchmark/vs-unpack.rb
 - ext/extconf.rb
+- lib/zscan/instructions.rb
 - lib/zscan.rb
+- spec/binary_scan_spec.rb
+- spec/combinator_spec.rb
+- spec/spec_helper.rb
+- spec/typed_scan_spec.rb
 - spec/zscan_spec.rb
 - ext/zscan.c
+- ext/bspec_exec.inc
+- ext/bspec_opcode_names.inc
 homepage: https://github.com/luikore/zscan
 licenses:
 - BSD