zscan 1.2 → 1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 71174140986194386fd74477637f21f95810d704
-  data.tar.gz: 72706410ead67f6cd9af96a7bf7024c72d7ef8c8
+  metadata.gz: 656c885fe2dcee21e852866ae02bf9fa960a3c9a
+  data.tar.gz: 1f5642a27ad99afc41494e556095fc46c98af22b
 SHA512:
-  metadata.gz: 7c9b9a9516dfa9b5ced99820f66976868d2c0a521df1eb35a517d262d3519601bb28b2a5e95797de411398b22bea3d9aa504d4af96f1634c1f9353d873d134d0
-  data.tar.gz: d3c3bcc5255f1909acafdf4bb1977d0ac75d73a6e837af0382cc5dd57d4a1e98443898bd49252e9d2d0b9ee3c7bc3eb0561c423bea4a91feb44bccf6138ac86f
+  metadata.gz: 844ac1574f62522bfa329e091f1a55558e17a2db7b0c39e5599fe4ad4038c66e563f1bd36d9ddf55132746c9265a2678f2abb2f0904c1fd5e3f4fec810d3197a
+  data.tar.gz: b156b693586831fcf094984583442ab146946fded1564b5d70f24d62db1cf72c3df5a91286a6e6a7822186f0655347e825a1d4f391c735157a2bf6121808d313

data/ext/bspec.c

@@ -1,6 +1,8 @@
 #include "zscan.h"
 
 static const rb_data_type_t* zscan_type;
+static void** bspec_opcodes;
+# include "bspec_init.inc"
 
 typedef struct {
   long s_size;
@@ -31,18 +33,31 @@ static VALUE bspec_alloc(VALUE klass) {
   bs->a_cap = 4;
   bs->a_size = 0;
   bs->code = (void**)malloc(bs->a_cap * sizeof(void*));
+  for (long i = 0; i < bs->a_cap; i++) {
+    bs->code[i] = bspec_opcodes[0];
+  }
   return TypedData_Wrap_Struct(klass, &bspec_type, bs);
 }
 
-static VALUE bspec_append(VALUE self, VALUE v_code, VALUE v_s_size) {
+static VALUE bspec_append(VALUE self, VALUE v_i) {
   BSpec* bs = rb_check_typeddata(self, &bspec_type);
-  if (bs->a_size == bs->a_cap) {
+  long i = NUM2LONG(v_i);
+  if (i < 0 || i >= bspec_opcodes_size) {
+    rb_raise(rb_eArgError, "bad opcode index");
+  }
+
+  // ensure size
+  if (bs->a_size == bs->a_cap - 1) { // end with 0:RET, always terminate
+    bs->code = (void**)realloc(bs->code, bs->a_cap * 2 * sizeof(void*));
+    long j = bs->a_cap;
     bs->a_cap *= 2;
-    bs->code = (void**)realloc(bs->code, bs->a_cap * sizeof(void*));
+    for (; j < bs->a_cap; j++) {
+      bs->code[j] = bspec_opcodes[0];
+    }
   }
-  long s_size = NUM2LONG(v_s_size);
-  bs->code[bs->a_size++] = ((void**)RSTRING_PTR(v_code))[0];
-  bs->s_size += s_size;
+
+  bs->code[bs->a_size++] = bspec_opcodes[i];
+  bs->s_size += bspec_s_sizes[i];
   return self;
 }
 
@@ -101,6 +116,23 @@ static VALUE bspec_big_endian_p(VALUE self) {
 
 #include "bspec_exec.inc"
 
+static VALUE bspec_opcodes_to_a(VALUE bspec) {
+  volatile VALUE a = rb_ary_new();
+  for (long i = 0; i < bspec_opcodes_size; i++) {
+    rb_ary_push(a, UINT64toNUM((uint64_t)(bspec_opcodes[i])));
+  }
+  return a;
+}
+
+static VALUE bspec_inspect_opcodes(VALUE bspec, VALUE self) {
+  BSpec* bs = rb_check_typeddata(self, &bspec_type);
+  volatile VALUE a = rb_ary_new();
+  for (long i = 0; i <= bs->a_size; i++) {
+    rb_ary_push(a, UINT64toNUM((uint64_t)(bs->code[i])));
+  }
+  return a;
+}
+
 static VALUE zscan_scan_binary(VALUE self, VALUE spec) {
   ZScan* p = rb_check_typeddata(self, zscan_type);
   if (!rb_enc_str_asciicompat_p(p->s)) {
@@ -126,16 +158,11 @@ void Init_zscan_bspec(VALUE zscan, const rb_data_type_t* _zscan_type) {
   zscan_type = _zscan_type;
   rb_define_method(zscan, "scan_binary", zscan_scan_binary, 1);
 
+  bspec_opcodes = (void**)bspec_exec(NULL, NULL, Qnil);
   VALUE bs = rb_define_class_under(zscan, "BinarySpec", rb_cObject);
   rb_define_singleton_method(bs, "big_endian?", bspec_big_endian_p, 0);
+  rb_define_singleton_method(bs, "_opcodes_to_a", bspec_opcodes_to_a, 0);
+  rb_define_singleton_method(bs, "_inspect_opcodes", bspec_inspect_opcodes, 1);
   rb_define_alloc_func(bs, bspec_alloc);
-  rb_define_method(bs, "append", bspec_append, 2);
-
-# include "bspec_opcode_names.inc"
-  void** opcodes = (void**)bspec_exec(NULL, NULL, Qnil);
-  for (long i = 0; i < bspec_opcode_size; i++) {
-    VALUE bytecode = rb_str_new((char*)&opcodes[i], sizeof(void*));
-    OBJ_FREEZE(bytecode);
-    rb_define_const(bs, bspec_opcode_names[i], bytecode);
-  }
+  rb_define_method(bs, "append", bspec_append, 1);
 }

data/ext/bspec_init.inc

@@ -0,0 +1,3 @@
+// GENERATED WITH: rake gen
+static const long bspec_s_sizes[] = {0, 1, 2, 2, 4, 4, 8, 8, 1, 2, 2, 4, 4, 8, 8, 4, 4, 8, 8};
+static const long bspec_opcodes_size = 19;

data/ext/zscan.c

@@ -69,10 +69,10 @@ static VALUE zscan_advance(VALUE self, VALUE v_diff) {
     long byteend = RSTRING_LEN(p->s);
     char* ptr = RSTRING_PTR(p->s);
     for (; p->pos < n && p->bytepos < byteend;) {
-      int n = rb_enc_mbclen(ptr + p->bytepos, ptr + byteend, enc);
-      if (n) {
+      int m = rb_enc_mbclen(ptr + p->bytepos, ptr + byteend, enc);
+      if (m) {
         p->pos++;
-        p->bytepos += n;
+        p->bytepos += m;
       } else {
         break;
       }
@@ -141,6 +141,35 @@ static VALUE zscan_eos_p(VALUE self) {
   return (p->bytepos == RSTRING_LEN(p->s) ? Qtrue : Qfalse);
 }
 
+static VALUE zscan_rest(VALUE self) {
+  P;
+  return rb_funcall(p->s, rb_intern("byteslice"), 2, LONG2NUM(p->bytepos), LONG2NUM(RSTRING_LEN(p->s)));
+}
+
+static VALUE zscan_rest_size(VALUE self) {
+  P;
+  rb_encoding* enc = rb_enc_get(p->s);
+  char* ptr = RSTRING_PTR(p->s) + p->bytepos;
+  long len = RSTRING_LEN(p->s) - p->bytepos;
+
+  long sz = 0;
+  for (long i = 0; i < len;) {
+    long n = rb_enc_mbclen(ptr + i, ptr + len, enc);
+    if (n) {
+      sz++;
+      i += n;
+    } else {
+      rb_raise(rb_eRuntimeError, "failed to scan char");
+    }
+  }
+  return LONG2NUM(sz);
+}
+
+static VALUE zscan_rest_bytesize(VALUE self) {
+  P;
+  return LONG2NUM(RSTRING_LEN(p->s) - p->bytepos);
+}
+
 regex_t *rb_reg_prepare_re(VALUE re, VALUE str);
 static VALUE zscan_match_bytesize(VALUE self, VALUE pattern) {
   P;
@@ -385,6 +414,10 @@ void Init_zscan() {
   rb_define_method(zscan, "bytepos=", zscan_bytepos_eq, 1);
   rb_define_method(zscan, "advance", zscan_advance, 1);
   rb_define_method(zscan, "eos?", zscan_eos_p, 0);
+  rb_define_method(zscan, "rest", zscan_rest, 0);
+  rb_define_method(zscan, "rest_size", zscan_rest_size, 0);
+  rb_define_method(zscan, "rest_bytesize", zscan_rest_bytesize, 0);
+
   rb_define_method(zscan, "match_bytesize", zscan_match_bytesize, 1);
   rb_define_method(zscan, "scan", zscan_scan, 1);
   rb_define_method(zscan, "push", zscan_push, 0);

data/lib/zscan.rb

@@ -3,7 +3,7 @@ require_relative "zscan/instructions"
 require "date"
 
 class ZScan
-  VERSION = '1.2'
+  VERSION = '1.3'
 
   def initialize s, dup=false
     if s.encoding.ascii_compatible?
@@ -98,10 +98,6 @@ class ZScan
     advance new_pos - pos
   end
 
-  def rest
-    _internal_string.byteslice bytepos, _internal_string.bytesize
-  end
-
   def reset
     self.pos = 0
     self
@@ -138,10 +134,21 @@ class ZScan
     _internal_string.byteslice(0, bytepos).count "\n"
   end
 
+  def slice size
+    r = _internal_string.slice pos, size
+    advance size
+    r
+  end
+
+  def byteslice bytesize
+    r = _internal_string.byteslice bytepos, bytesize
+    self.bytepos += bytesize
+    r
+  end
+
   def self.binary_spec &p
     bs = BinarySpec.send :new
     bs.instance_eval &p
-    bs.send :append, BinarySpec::RET, 0
     bs
   end
 

data/lib/zscan/instructions.rb

@@ -3,109 +3,109 @@ class ZScan::BinarySpec
   def int8 n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append INT8, 1
+      append 1
     end
   end
   def int16 n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append INT16, 2
+      append 2
     end
   end
   def int16_swap n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append INT16_SWAP, 2
+      append 3
     end
   end
   def int32 n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append INT32, 4
+      append 4
     end
   end
   def int32_swap n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append INT32_SWAP, 4
+      append 5
     end
   end
   def int64 n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append INT64, 8
+      append 6
     end
   end
   def int64_swap n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append INT64_SWAP, 8
+      append 7
     end
   end
   def uint8 n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append UINT8, 1
+      append 8
     end
   end
   def uint16 n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append UINT16, 2
+      append 9
     end
   end
   def uint16_swap n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append UINT16_SWAP, 2
+      append 10
     end
   end
   def uint32 n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append UINT32, 4
+      append 11
     end
   end
   def uint32_swap n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append UINT32_SWAP, 4
+      append 12
     end
   end
   def uint64 n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append UINT64, 8
+      append 13
     end
   end
   def uint64_swap n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append UINT64_SWAP, 8
+      append 14
     end
   end
   def single n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append SINGLE, 4
+      append 15
     end
   end
   def single_swap n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append SINGLE_SWAP, 4
+      append 16
     end
   end
   def double n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append DOUBLE, 8
+      append 17
     end
   end
   def double_swap n=1
     raise ArgumentError, "repeat count should be >= 1, but got #{n}" if n < 1
     n.times do
-      append DOUBLE_SWAP, 8
+      append 18
     end
   end
   if ZScan::BinarySpec.big_endian?

data/rakefile

@@ -86,7 +86,7 @@ file gem_package => gem_files do
 end
 
 desc "generate files"
-task :gen => %w[ext/bspec_exec.inc ext/bspec_opcode_names.inc lib/zscan/instructions.rb]
+task :gen => %w[ext/bspec_exec.inc ext/bspec_init.inc lib/zscan/instructions.rb]
 
 file 'ext/bspec_exec.inc' => __FILE__ do
   puts "generating ext/bspec_exec.inc"
@@ -122,13 +122,13 @@ BS_RET:
   end
 end
 
-file 'ext/bspec_opcode_names.inc' => __FILE__ do
-  puts 'generating ext/bspec_opcode_names.inc'
-  opcode_names = bspec_insns.map(&:inspect).join ', '
-  File.open 'ext/bspec_opcode_names.inc', 'w' do |f|
+file 'ext/bspec_init.inc' => __FILE__ do
+  puts 'generating ext/bspec_init.inc'
+  opcode_incrs = bspec_insns.map{|ins| bspec_incr ins}.join ', '
+  File.open 'ext/bspec_init.inc', 'w' do |f|
     f.puts "// GENERATED WITH: rake gen"
-    f.puts %Q|const char* bspec_opcode_names[] = {"RET", #{opcode_names}};|
-    f.puts %Q|long bspec_opcode_size = #{bspec_insns.size + 1};|
+    f.puts %Q|static const long bspec_s_sizes[] = {0, #{opcode_incrs}};|
+    f.puts %Q|static const long bspec_opcodes_size = #{bspec_insns.size + 1};|
   end
 end
 
@@ -138,12 +138,12 @@ file 'lib/zscan/instructions.rb' => __FILE__ do
     f.puts "# GENERATED WITH: rake gen"
     f.puts "class ZScan::BinarySpec"
 
-    bspec_insns.each do |ins|
+    bspec_insns.each_with_index do |ins, i|
       f.puts <<-RUBY
   def #{ins.downcase} n=1
     raise ArgumentError, "repeat count should be >= 1, but got \#{n}" if n < 1
     n.times do
-      append #{ins}, #{bspec_incr ins}
+      append #{i + 1}
     end
   end
       RUBY

data/readme.md

@@ -59,6 +59,8 @@ See also https://bugs.ruby-lang.org/issues/7092
 - `#scan regexp_or_string`
 - `#skip regexp_or_string`
 - `#match_bytesize regexp_or_string` return length of matched bytes or `nil`.
+- `#slice n` slice sub string of n chars from current pos, advances the cursor.
+- `#byteslice n` slice sub string of n bytes from cursor pos, advances the cursor.
 - `#scan_float` scan a float number which is not starting with space. It deals with multibyte encodings for you.
 - `#scan_int radix=nil` if radix is nil, decide base by prefix: `0x` is 16, `0` is 8, `0b` is 2, otherwise 10. `radix` should be in range `2..36`.
 - `#scan_date format_string, start=Date::ITALY` scan a `DateTime` object, see also [strptime](http://rubydoc.info/stdlib/date/DateTime.strptime).
@@ -66,7 +68,9 @@ See also https://bugs.ruby-lang.org/issues/7092
 - `#unpack format_string`
 - `#eos?`
 - `#string` note: return a dup. Don't worry the performance because it is a copy-on-write string.
-- `#rest`
+- `#rest` rest unscanned sub string.
+- `#rest_size` count characters of unscanned sub string.
+- `#rest_bytesize` count bytes of unscanned sub string.
 
 ## String delegates
 

data/spec/zscan_spec.rb

@@ -21,10 +21,19 @@ describe ZScan do
     assert_equal 8, @z.bytepos
   end
 
+  it "slices a certain bytes or chars" do
+    assert_equal 'ab', (@z.slice 2)
+    assert_equal '你', (@z.slice 1)
+    assert_equal '好', (@z.byteslice 3)
+    assert_equal true, @z.eos?
+  end
+
   it "scans from middle" do
     @z.bytepos = 2
     assert_equal '你', @z.scan('你')
     assert_equal '好', @z.rest
+    assert_equal '好'.size, @z.rest_size
+    assert_equal '好'.bytesize, @z.rest_bytesize
   end
 
   it "won't overflow pos" do

data/zscan.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = "zscan"
-  s.version = "1.2" # version mapped from zscan.rb, don't change here
+  s.version = "1.3" # version mapped from zscan.rb, don't change here
   s.author = "Zete Lui"
   s.homepage = "https://github.com/luikore/zscan"
   s.platform = Gem::Platform::RUBY

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zscan
 version: !ruby/object:Gem::Version
-  version: '1.2'
+  version: '1.3'
 platform: ruby
 authors:
 - Zete Lui
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-17 00:00:00.000000000 Z
+date: 2013-05-18 00:00:00.000000000 Z
 dependencies: []
 description: improved string scanner, respects anchors and lookbehinds, supports codepoint
   positioning
@@ -36,7 +36,7 @@ files:
 - ext/bspec.c
 - ext/zscan.c
 - ext/bspec_exec.inc
-- ext/bspec_opcode_names.inc
+- ext/bspec_init.inc
 homepage: https://github.com/luikore/zscan
 licenses:
 - BSD

data/ext/bspec_opcode_names.inc

@@ -1,3 +0,0 @@
-// GENERATED WITH: rake gen
-const char* bspec_opcode_names[] = {"RET", "INT8", "INT16", "INT16_SWAP", "INT32", "INT32_SWAP", "INT64", "INT64_SWAP", "UINT8", "UINT16", "UINT16_SWAP", "UINT32", "UINT32_SWAP", "UINT64", "UINT64_SWAP", "SINGLE", "SINGLE_SWAP", "DOUBLE", "DOUBLE_SWAP"};
-long bspec_opcode_size = 19;