zscan 1.3 → 2.0

data/ext/zscan.c

@@ -46,7 +46,7 @@ static VALUE zscan_internal_string(VALUE self) {
 
 static VALUE zscan_pos(VALUE self) {
   P;
-  return ULONG2NUM(p->pos);
+  return LONG2NUM(p->pos);
 }
 
 static VALUE zscan_advance(VALUE self, VALUE v_diff) {
@@ -83,10 +83,10 @@ static VALUE zscan_advance(VALUE self, VALUE v_diff) {
 
 static VALUE zscan_bytepos(VALUE self) {
   P;
-  return ULONG2NUM(p->bytepos);
+  return LONG2NUM(p->bytepos);
 }
 
-static VALUE zscan_bytepos_eq(VALUE self, VALUE v_bytepos) {
+VALUE zscan_bytepos_eq(VALUE self, VALUE v_bytepos) {
   P;
   long signed_bytepos = NUM2LONG(v_bytepos);
   long from, to, bytepos;
@@ -170,13 +170,22 @@ static VALUE zscan_rest_bytesize(VALUE self) {
   return LONG2NUM(RSTRING_LEN(p->s) - p->bytepos);
 }
 
+static VALUE zscan_cleanup(VALUE self) {
+  P;
+  long rest_len = RSTRING_LEN(p->s) - p->bytepos;
+  p->s = rb_funcall(p->s, rb_intern("byteslice"), 2, LONG2NUM(p->bytepos), LONG2NUM(rest_len));
+  p->bytepos = 0;
+  p->pos = 0;
+  return self;
+}
+
 regex_t *rb_reg_prepare_re(VALUE re, VALUE str);
 static VALUE zscan_match_bytesize(VALUE self, VALUE pattern) {
   P;
   if (TYPE(pattern) == T_STRING) {
-    volatile VALUE ss = rb_funcall(p->s, rb_intern("byteslice"), 2, ULONG2NUM(p->bytepos), ULONG2NUM(RSTRING_LEN(p->s)));
+    volatile VALUE ss = rb_funcall(p->s, rb_intern("byteslice"), 2, LONG2NUM(p->bytepos), LONG2NUM(RSTRING_LEN(p->s) - p->bytepos));
     if (RTEST(rb_funcall(ss, rb_intern("start_with?"), 1, pattern))) {
-      return ULONG2NUM(RSTRING_LEN(pattern));
+      return LONG2NUM(RSTRING_LEN(pattern));
     }
   } else if (TYPE(pattern) == T_REGEXP) {
     regex_t *re = rb_reg_prepare_re(pattern, p->s); // prepare with compatible encoding
@@ -228,6 +237,18 @@ static VALUE zscan_scan(VALUE self, VALUE pattern) {
   }
 }
 
+static VALUE zscan_skip(VALUE self, VALUE pattern) {
+  volatile VALUE v_bytelen = zscan_match_bytesize(self, pattern);
+  if (v_bytelen == Qnil) {
+    return Qnil;
+  } else {
+    P;
+    long bytepos = p->bytepos + NUM2LONG(v_bytelen);
+    zscan_bytepos_eq(self, LONG2NUM(bytepos));
+    return LONG2NUM(bytepos);
+  }
+}
+
 static VALUE zscan_push(VALUE self) {
   P;
   if (p->stack_i + 1 == p->stack_cap) {
@@ -401,7 +422,14 @@ VALUE zscan_scan_float(VALUE self) {
   }
 }
 
-extern void Init_zscan_bspec(VALUE, const rb_data_type_t*);
+VALUE zscan_unpack(VALUE self, VALUE fmt) {
+  P;
+  long parsed_len = 0;
+  volatile VALUE str = zscan_rest(self);
+  VALUE r = zscan_internal_unpack(str, fmt, &parsed_len);
+  zscan_bytepos_eq(self, LONG2NUM(p->bytepos + parsed_len));
+  return r;
+}
 
 void Init_zscan() {
   VALUE zscan = rb_define_class("ZScan", rb_cObject);
@@ -417,9 +445,12 @@ void Init_zscan() {
   rb_define_method(zscan, "rest", zscan_rest, 0);
   rb_define_method(zscan, "rest_size", zscan_rest_size, 0);
   rb_define_method(zscan, "rest_bytesize", zscan_rest_bytesize, 0);
+  rb_define_method(zscan, "cleanup", zscan_cleanup, 0);
 
   rb_define_method(zscan, "match_bytesize", zscan_match_bytesize, 1);
   rb_define_method(zscan, "scan", zscan_scan, 1);
+  rb_define_method(zscan, "skip", zscan_skip, 1);
+  rb_define_method(zscan, "unpack", zscan_unpack, 1);
   rb_define_method(zscan, "push", zscan_push, 0);
   rb_define_method(zscan, "pop", zscan_pop, 0);
   rb_define_method(zscan, "drop", zscan_drop, 0);

data/ext/zscan.h

@@ -17,3 +17,7 @@ typedef struct {
   long stack_cap;
   Pos* stack;
 } ZScan;
+
+VALUE zscan_bytepos_eq(VALUE self, VALUE v_bytepos);
+VALUE zscan_internal_unpack(VALUE str, VALUE fmt, long* parsed_len);
+void Init_zscan_bspec(VALUE, const rb_data_type_t*);

data/generate/bspec.rb

@@ -0,0 +1,48 @@
+class ZScan
+  class BSpec
+    def _append_expect expect_i, pack_format, opts
+      if opts.is_a?(Hash)
+        expect = opts[:expect]
+        if expect.is_a?(Integer) and opts.size == 1
+          ZScan::BSpec._append self, expect_i
+          ZScan::BSpec._append_expect self, [expect].pack(pack_format)
+        else
+          raise ArgumentError, "only :expect option allowed, but got #{opts.inspect}"
+        end
+      end
+    end
+
+    def initialize &p
+      instance_eval &p
+    end
+
+    <% @int_ins.each do |ins, i| %>
+    def <%= ins.downcase %> opts=nil
+      _append_expect <%= to_expect_i ins %>, <%= to_pack_format ins %>, opts
+      ZScan::BSpec._append self, <%= i %>
+    end
+    <% end %>
+
+    <% @float_ins.each do |ins, i| %>
+    def <%= ins.downcase %>
+      ZScan::BSpec._append self, <%= i %>
+    end
+    <% end %>
+
+    if ZScan::BSpec.big_endian?
+      <% @alias_ins.map(&:downcase).each do |ins| %>
+      alias <%= ins %>_be <%= ins %>
+      alias <%= ins %>_le <%= ins %>_swap
+      <% end %>
+    else
+      <% @alias_ins.map(&:downcase).each do |ins| %>
+      alias <%= ins %>_le <%= ins %>
+      alias <%= ins %>_be <%= ins %>_swap
+      <% end %>
+    end
+    alias byte uint8
+    alias float single
+    alias float_le single_le
+    alias float_be single_be
+  end
+end

data/generate/bspec_exec.inc

@@ -0,0 +1,29 @@
+__attribute__((__noinline__))
+static VALUE bspec_exec(void** ip, char* s, VALUE a) {
+  static void* opcodes[] = { <%= @opcode_list.join ', ' %> };
+  if (ip == NULL) {
+    return (VALUE)opcodes;
+  }
+  goto **(ip++);
+BS_RET:
+  return a;
+<% [8, 16, 32, 64].each do |len| %>
+BS_EXPECT<%= len %>:
+  {
+    char* expect = (char*)(ip++);
+    if (strncmp(s, expect, <%= len / 8 %>)) {
+      return Qnil;
+    }
+    goto **(ip++);
+  }
+<% end %>
+<% @conv_insns.each do |ins| %>
+BS_<%= ins %>:
+  {
+    <%= extract ins %>;
+    rb_ary_push(a, <%= convert ins %>);
+    s += <%= incr ins %>;
+    goto **(ip++);
+  }
+<% end %>
+}

data/generate/bspec_init.inc

@@ -0,0 +1,2 @@
+static const long bspec_s_sizes[] = {<%= @incrs %>};
+static const long bspec_opcodes_size = <%= @insns_size %>;

data/generate/generate.rb

@@ -0,0 +1,147 @@
+require "erb"
+
+class Generate
+  def self.files
+    %w[ext/bspec_exec.inc ext/bspec_init.inc lib/zscan/bspec.rb]
+  end
+
+  def self.generate file
+    new(file).render
+  end
+
+  def initialize file
+    case file
+    when 'ext/bspec_exec.inc'
+      generate_exec
+    when 'ext/bspec_init.inc'
+      generate_init
+    when 'lib/zscan/bspec.rb'
+      generate_rb
+    else
+      raise "unkown file: #{file}"
+    end
+    @file = file
+  end
+
+  def render
+    ERB.new(File.read File.join(__dir__, File.basename(@file))).result binding
+  end
+
+  def generate_init
+    @incrs = insns.map{|ins| incr ins}.join ', '
+    @insns_size = insns.size
+  end
+
+  def generate_exec
+    @opcode_list = insns.map{|ins| "&&BS_#{ins}" }
+    @conv_insns = insns.select{|ins| ins !~ /RET|EXPECT/ }
+  end
+
+  def generate_rb
+    groups = insns.each_with_index.to_a.group_by{|x|
+      case x.first
+      when /INT/
+        :int
+      when /RET|EXPECT/
+        :misc
+      else
+        :float
+      end
+    }
+    @int_ins = groups[:int]
+    @float_ins = groups[:float]
+    @alias_ins = swap_types.map &:downcase
+  end
+
+  def swap_types
+    %w[INT16 INT32 INT64 UINT16 UINT32 UINT64 SINGLE DOUBLE]
+  end
+
+  def insns
+    ['RET', 'EXPECT8', 'EXPECT16', 'EXPECT32', 'EXPECT64', 'INT8', 'UINT8', *swap_types.flat_map{|ty| [ty, "#{ty}_SWAP"] }]
+  end
+
+  def to_pack_format ins
+    raise "bad" if ins !~ /INT/
+    base = {
+      '8' => 'c',
+      '16' => 's',
+      '32' => 'l',
+      '64' => 'q'
+    }[ins[/\d+/]]
+    if ins.start_with?('U')
+      base.upcase!
+    end
+
+    if ins[/\d+/] == '8'
+      return "'#{base}'"
+    end
+
+    if ins.end_with?('SWAP')
+      format = base + '<'
+      swap_format = base + '>'
+    else
+      format = base + '>'
+      swap_format = base + '<'
+    end
+    "(BSpec.big_endian? ? '#{format}' : '#{swap_format}')"
+  end
+
+  def to_expect_i ins
+    bits = ins[/\d+/]
+    insns.index "EXPECT#{bits}"
+  end
+
+  # following methods used in C-code gen
+
+  def incr ins
+    case ins
+    when /INT(\d+)/; $1.to_i / 8
+    when /SINGLE/; 4
+    when /DOUBLE/; 8
+    when /RET|EXPECT/; 0
+    else; raise 'bad'
+    end
+  end
+
+  def c_type ins
+    case ins
+    when /(U?INT\d+)/; "#{$1.downcase}_t"
+    when /SINGLE/; 'float'
+    when /DOUBLE/; 'double'
+    else; raise 'bad'
+    end
+  end
+
+  def extract ins
+    type = c_type ins
+    len = incr(ins) * 8
+    r = "((uint#{len}_t*)s)[0]"
+    if ins.end_with?('SWAP')
+      r = "swap#{len}(#{r})"
+    end
+    "uint#{len}_t r = #{r}"
+  end
+
+  def convert ins
+    case ins
+    when /(U)?INT64|UINT32/
+      if ins.start_with?('U')
+        "UINT64toNUM(r)"
+      else
+        "INT64toNUM(CAST(r, int64_t))"
+      end
+    when /INT32/
+      "INT2NUM(CAST(r, int32_t))"
+    when /INT(16|8)/
+      "INT2FIX(CAST(r, #{c_type ins}))"
+    when /SINGLE/
+      "DBL2NUM((double)CAST(r, float))"
+    when /DOUBLE/
+      "DBL2NUM(CAST(r, double))"
+    else
+      raise 'bad'
+    end
+  end
+
+end

data/lib/zscan.rb

@@ -1,9 +1,9 @@
 require_relative "../ext/zscan"
-require_relative "zscan/instructions"
+require_relative "zscan/bspec"
 require "date"
 
 class ZScan
-  VERSION = '1.3'
+  VERSION = '2.0'
 
   def initialize s, dup=false
     if s.encoding.ascii_compatible?
@@ -18,12 +18,6 @@ class ZScan
     _internal_string.dup
   end
 
-  def skip re_or_str
-    if sz = match_bytesize(re_or_str)
-      self.bytepos += sz
-    end
-  end
-
   def scan_int radix=nil
     negative = false
     r = try do
@@ -44,7 +38,7 @@ class ZScan
         case radix
         when 2;  /[01]+/
         when 8;  /[0-7]+/
-        when 10; /\d+/
+        when 10; /[0-9]+/ # don't use \d because it matches unicode numbers
         when 16; /\h+/i
         else
           if radix < 10
@@ -53,7 +47,7 @@ class ZScan
             raise ArgumentError, "invalid radix #{radix}"
           else
             end_char = ('a'.ord + (radix - 11)).chr
-            /[\da-#{end_char}]+/i
+            /[0-9a-#{end_char}]+/i
           end
         end
     end
@@ -80,20 +74,6 @@ class ZScan
     end
   end
 
-  def unpack format
-    if format.index('@')
-      raise ArgumentError, 'position instruction @ not supported'
-    end
-    r = rest.unpack format
-    if r.index(nil)
-      return
-    end
-    # XXX pack to get parsed length because no related API is exposed ...
-    len = r.pack(format).bytesize
-    self.bytepos += len
-    r
-  end
-
   def pos= new_pos
     advance new_pos - pos
   end
@@ -146,19 +126,6 @@ class ZScan
     r
   end
 
-  def self.binary_spec &p
-    bs = BinarySpec.send :new
-    bs.instance_eval &p
-    bs
-  end
-
-  class BinarySpec
-    private :append
-    class << self
-      private :new
-    end
-  end
-
   private :_internal_init, :_internal_string
 end
 

data/lib/zscan/bspec.rb

@@ -0,0 +1,168 @@
+# generated by rake gen
+class ZScan
+  class BSpec
+    def _append_expect expect_i, pack_format, opts
+      if opts.is_a?(Hash)
+        expect = opts[:expect]
+        if expect.is_a?(Integer) and opts.size == 1
+          ZScan::BSpec._append self, expect_i
+          ZScan::BSpec._append_expect self, [expect].pack(pack_format)
+        else
+          raise ArgumentError, "only :expect option allowed, but got #{opts.inspect}"
+        end
+      end
+    end
+
+    def initialize &p
+      instance_eval &p
+    end
+
+    
+    def int8 opts=nil
+      _append_expect 1, 'c', opts
+      ZScan::BSpec._append self, 5
+    end
+    
+    def uint8 opts=nil
+      _append_expect 1, 'C', opts
+      ZScan::BSpec._append self, 6
+    end
+    
+    def int16 opts=nil
+      _append_expect 2, (BSpec.big_endian? ? 's>' : 's<'), opts
+      ZScan::BSpec._append self, 7
+    end
+    
+    def int16_swap opts=nil
+      _append_expect 2, (BSpec.big_endian? ? 's<' : 's>'), opts
+      ZScan::BSpec._append self, 8
+    end
+    
+    def int32 opts=nil
+      _append_expect 3, (BSpec.big_endian? ? 'l>' : 'l<'), opts
+      ZScan::BSpec._append self, 9
+    end
+    
+    def int32_swap opts=nil
+      _append_expect 3, (BSpec.big_endian? ? 'l<' : 'l>'), opts
+      ZScan::BSpec._append self, 10
+    end
+    
+    def int64 opts=nil
+      _append_expect 4, (BSpec.big_endian? ? 'q>' : 'q<'), opts
+      ZScan::BSpec._append self, 11
+    end
+    
+    def int64_swap opts=nil
+      _append_expect 4, (BSpec.big_endian? ? 'q<' : 'q>'), opts
+      ZScan::BSpec._append self, 12
+    end
+    
+    def uint16 opts=nil
+      _append_expect 2, (BSpec.big_endian? ? 'S>' : 'S<'), opts
+      ZScan::BSpec._append self, 13
+    end
+    
+    def uint16_swap opts=nil
+      _append_expect 2, (BSpec.big_endian? ? 'S<' : 'S>'), opts
+      ZScan::BSpec._append self, 14
+    end
+    
+    def uint32 opts=nil
+      _append_expect 3, (BSpec.big_endian? ? 'L>' : 'L<'), opts
+      ZScan::BSpec._append self, 15
+    end
+    
+    def uint32_swap opts=nil
+      _append_expect 3, (BSpec.big_endian? ? 'L<' : 'L>'), opts
+      ZScan::BSpec._append self, 16
+    end
+    
+    def uint64 opts=nil
+      _append_expect 4, (BSpec.big_endian? ? 'Q>' : 'Q<'), opts
+      ZScan::BSpec._append self, 17
+    end
+    
+    def uint64_swap opts=nil
+      _append_expect 4, (BSpec.big_endian? ? 'Q<' : 'Q>'), opts
+      ZScan::BSpec._append self, 18
+    end
+    
+
+    
+    def single
+      ZScan::BSpec._append self, 19
+    end
+    
+    def single_swap
+      ZScan::BSpec._append self, 20
+    end
+    
+    def double
+      ZScan::BSpec._append self, 21
+    end
+    
+    def double_swap
+      ZScan::BSpec._append self, 22
+    end
+    
+
+    if ZScan::BSpec.big_endian?
+      
+      alias int16_be int16
+      alias int16_le int16_swap
+      
+      alias int32_be int32
+      alias int32_le int32_swap
+      
+      alias int64_be int64
+      alias int64_le int64_swap
+      
+      alias uint16_be uint16
+      alias uint16_le uint16_swap
+      
+      alias uint32_be uint32
+      alias uint32_le uint32_swap
+      
+      alias uint64_be uint64
+      alias uint64_le uint64_swap
+      
+      alias single_be single
+      alias single_le single_swap
+      
+      alias double_be double
+      alias double_le double_swap
+      
+    else
+      
+      alias int16_le int16
+      alias int16_be int16_swap
+      
+      alias int32_le int32
+      alias int32_be int32_swap
+      
+      alias int64_le int64
+      alias int64_be int64_swap
+      
+      alias uint16_le uint16
+      alias uint16_be uint16_swap
+      
+      alias uint32_le uint32
+      alias uint32_be uint32_swap
+      
+      alias uint64_le uint64
+      alias uint64_be uint64_swap
+      
+      alias single_le single
+      alias single_be single_swap
+      
+      alias double_le double
+      alias double_be double_swap
+      
+    end
+    alias byte uint8
+    alias float single
+    alias float_le single_le
+    alias float_be single_be
+  end
+end