zscan 0.5 → 1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 743c9b854996ff1714f83f805a8d975b261993b5
-  data.tar.gz: 65254b0e114cd41b727e5481da48704b28ee2bf1
+  metadata.gz: c343abbf9b043bcb7643ac193bb5f9a0fd76245e
+  data.tar.gz: fe57d1e6e352cad9d9ac2f8095afdbbd4affbebe
 SHA512:
-  metadata.gz: 028d305bcee7917a29caadbe8a4ff1254e682de582c5316ba0267d95ab0dd5fe01a47198fc5fda3aa5ba0e6c66ccd813d17fa464986c25bf528c3fc460a1e55d
-  data.tar.gz: eb2f9bff39078f3a835868147f3e36e388544ea6a939a053b0db3298cfef8d56945b06a78fba102b9b04345fef5ac5464e70d16f2999a25cd72f16c6489268ad
+  metadata.gz: 687a22c21f5da837039b557365807a89b5afd33cea5301baa1d5321c3a5b324f63f7b96524166f388da1c296e2e775ce8007e97281ad084c1583aa20c247ff6f
+  data.tar.gz: d6fde3485b2bb8e4db9bb80513c370dec17e331c1e20c5eb087742d86fdb06c403136b98be3e14dc65747b897182f4eaa0132ebc454b427e1e5b1cc622a1c8e0

data/ext/zscan.c

@@ -43,8 +43,8 @@ static VALUE zscan_alloc(VALUE klass) {
   ZScan* p = ALLOC(ZScan);
   MEMZERO(p, ZScan, 1);
   p->s = Qnil;
-  p->stack_cap = 5;
-  p->stack = (Pos*)malloc(sizeof(Pos) * 5);
+  p->stack_cap = 8;
+  p->stack = (Pos*)malloc(sizeof(Pos) * p->stack_cap);
   return TypedData_Wrap_Struct(klass, &zscan_type, p);
 }
 
@@ -158,10 +158,10 @@ static VALUE zscan_eos_p(VALUE self) {
 }
 
 regex_t *rb_reg_prepare_re(VALUE re, VALUE str);
-static VALUE zscan_bmatch_p(VALUE self, VALUE pattern) {
+static VALUE zscan_match_bytesize(VALUE self, VALUE pattern) {
   P;
   if (TYPE(pattern) == T_STRING) {
-    volatile VALUE ss = rb_funcall(self, rb_intern("rest"), 0);
+    volatile VALUE ss = rb_funcall(p->s, rb_intern("byteslice"), 2, ULONG2NUM(p->bytepos), ULONG2NUM(RSTRING_LEN(p->s)));
     if (RTEST(rb_funcall(ss, rb_intern("start_with?"), 1, pattern))) {
       return ULONG2NUM(RSTRING_LEN(pattern));
     }
@@ -201,10 +201,10 @@ static VALUE zscan_bmatch_p(VALUE self, VALUE pattern) {
   return Qnil;
 }
 
-static VALUE zscan_push_pos(VALUE self) {
+static VALUE zscan_push(VALUE self) {
   P;
   if (p->stack_i + 1 == p->stack_cap) {
-    p->stack_cap *= 2;
+    p->stack_cap = p->stack_cap * 1.4 + 3;
     p->stack = (Pos*)realloc(p->stack, sizeof(Pos) * p->stack_cap);
   }
   Pos e = {p->pos, p->bytepos};
@@ -212,7 +212,7 @@ static VALUE zscan_push_pos(VALUE self) {
   return self;
 }
 
-static VALUE zscan_pop_pos(VALUE self) {
+static VALUE zscan_pop(VALUE self) {
   P;
   if (p->stack_i) {
     p->pos = p->stack[p->stack_i].pos;
@@ -225,7 +225,7 @@ static VALUE zscan_pop_pos(VALUE self) {
   return self;
 }
 
-static VALUE zscan_drop_top(VALUE self) {
+static VALUE zscan_drop(VALUE self) {
   P;
   if (p->stack_i) {
     p->stack_i--;
@@ -233,7 +233,7 @@ static VALUE zscan_drop_top(VALUE self) {
   return self;
 }
 
-static VALUE zscan_resume_top(VALUE self) {
+static VALUE zscan_restore(VALUE self) {
   P;
   if (p->stack_i) {
     p->pos = p->stack[p->stack_i].pos;
@@ -253,9 +253,9 @@ void Init_zscan() {
   rb_define_method(zscan, "bytepos=", zscan_bytepos_eq, 1);
   rb_define_method(zscan, "advance", zscan_advance, 1);
   rb_define_method(zscan, "eos?", zscan_eos_p, 0);
-  rb_define_method(zscan, "bmatch?", zscan_bmatch_p, 1);
-  rb_define_method(zscan, "push_pos", zscan_push_pos, 0);
-  rb_define_method(zscan, "pop_pos", zscan_pop_pos, 0);
-  rb_define_method(zscan, "drop_top", zscan_drop_top, 0);
-  rb_define_method(zscan, "resume_top", zscan_resume_top, 0);
+  rb_define_method(zscan, "match_bytesize", zscan_match_bytesize, 1);
+  rb_define_method(zscan, "push", zscan_push, 0);
+  rb_define_method(zscan, "pop", zscan_pop, 0);
+  rb_define_method(zscan, "drop", zscan_drop, 0);
+  rb_define_method(zscan, "restore", zscan_restore, 0);
 }

data/lib/zscan.rb

@@ -1,7 +1,7 @@
 require_relative "../ext/zscan"
 
 class ZScan
-  VERSION = '0.5'
+  VERSION = '1.0'
 
   def initialize s, dup=false
     _internal_init dup ? s.dup : s
@@ -12,7 +12,7 @@ class ZScan
   end
 
   def scan re_or_str
-    if sz = bmatch?(re_or_str)
+    if sz = match_bytesize(re_or_str)
       r = _internal_string.byteslice bytepos, sz
       self.bytepos += sz
       r
@@ -20,7 +20,7 @@ class ZScan
   end
 
   def skip re_or_str
-    if sz = bmatch?(re_or_str)
+    if sz = match_bytesize(re_or_str)
       self.bytepos += sz
     end
   end
@@ -33,5 +33,14 @@ class ZScan
     _internal_string.byteslice bytepos, _internal_string.bytesize
   end
 
+  def reset
+    self.pos = 0
+    self
+  end
+
+  def terminate
+    self.pos = _internal_string.size
+  end
+
   private :_internal_init, :_internal_string
 end

data/rakefile

@@ -0,0 +1,34 @@
+Dir.chdir __dir__
+version_re  = /\d+(\.\d+)*/
+version     = `command grep 'VERSION =' lib/zscan.rb`[version_re]
+gem_files   = Dir.glob('{rakefile,zscan.gemspec,readme.md,**/*.{rb,c}}')
+gem_package = "zscan-#{version}.gem"
+
+desc "build and test"
+task :default => [:test, gem_package]
+
+desc "build and run test"
+task :test do
+  Dir.chdir "ext"
+  sh "make"
+  Dir.chdir ".."
+  sh "rspec"
+end
+
+desc "pack gem"
+file gem_package => gem_files do
+  sh "rm zscan-*.gem"
+
+  new_version = false
+  lines = File.readlines('zscan.gemspec')
+  lines.each do |line|
+    if line =~ /s\.version =/ and (line.sub! version_re, version)
+      new_version = true
+      break
+    end
+  end
+  if new_version
+    File.open('zscan.gemspec', 'w'){|f| f << lines.join }
+  end
+  sh "gem build zscan.gemspec"
+end

data/readme.md

@@ -1,24 +1,30 @@
-## Motivation
-
-A simple string scanner. Provides... much less methods than `StringScanner`.
+## Features
 
-It supports either string or regexp as scan param.
+- `ZScan#scan`/`ZScan#skip`/`ZScan#match_bytesize` accept either string or regexp as param.
+- `ZScan#pos` is the codepoint position, and `ZScan#bytepos` is byte position.
+- Correctly scans anchors and look behind predicates.
+- Pos stack manipulation.
 
-`pos` is by codepoints instead of bytes, use `bytepos` to locate byte position.
+## Install
 
-It provides a position stack for you to efficiently manage scanning locations.
+```bash
+gem ins zscan
+```
 
-It correctly scans anchors. The following codes demonstrate the behavior:
+## Typical use
 
-```ruby
+``` ruby
 require 'zscan'
-z = ZScan.new 'ab'
-z.pos = 1
-z.scan /(?<a)/ #=> ''
-z.scan /^/     #=> nil
+z = ZScan.new 'hello world'
+z.scan 'hello' #=> 'hello'
+z.skip ' '
+z.scan /\w+/   #=> 'world'
+z.eos?         #=> true
 ```
 
-While with `StringScanner`:
+## Motivation
+
+Ruby's stdlib `StringScanner` treats the scanning position as beginning of string:
 
 ```ruby
 require 'strscan'
@@ -28,29 +34,62 @@ s.scan /(?<a)/ #=> nil
 s.scan /^/     #=> ''
 ```
 
+But for building parser generators, I need the scanner check the whole string for anchors and lookbehinds:
+
+```ruby
+require 'zscan'
+z = ZScan.new 'ab'
+z.pos = 1
+z.scan /(?<a)/ #=> ''
+z.scan /^/     #=> nil
+```
+
 See also https://bugs.ruby-lang.org/issues/7092
 
 ## Methods
 
 - `ZScan.new string, dup=false`
-- `scan regexp_or_string`
-- `skip regexp_or_string`
-- `bmatch? regexp_or_string` returns length of matched bytes or nil
-- `eos?`
-- `string` note: returns a COW dup
-- `rest`
+- `#scan regexp_or_string`
+- `#skip regexp_or_string`
+- `#match_bytesize regexp_or_string` returns length of matched bytes or nil
+- `#eos?`
+- `#string` note: returns a COW dup
+- `#rest`
 
-## Position management
+## Pos management
 
-- `pos`
-- `pos= new_pos` note: complexity ~ `new_pos > pos ? new_pos - pos : new_pos`.
-- `bytepos`
-- `bytepos= new_bytepos` note: complexity ~ `abs(new_bytepos - bytepos)`.
-- `advance n` move forward `n` codepoints, if `n < 0`, move backward. Stops at beginning or end.
+- `#pos`
+- `#pos= new_pos` note: complexity ~ `new_pos > pos ? new_pos - pos : new_pos`.
+- `#bytepos`
+- `#bytepos= new_bytepos` note: complexity ~ `abs(new_bytepos - bytepos)`.
+- `#advance n` move forward `n` codepoints, if `n < 0`, move backward. Stops at beginning or end.
+- `#reset` go to beginning.
+- `#terminate` go to end of string.
 
 ## Efficient pos stack manipulation
 
-- `push_pos` pushes current pos into the stack.
-- `pop_pos` sets current pos to top of the stack, and pops it.
-- `drop_top` drops top of pos stack without changing current pos.
-- `resume_top` sets current pos to top of the stack.
+- `#push` pushes current pos into the stack.
+- `#pop` sets current pos to top of the stack, and pops it.
+- `#drop` drops top of pos stack without changing current pos.
+- `#restore` sets current pos to top of the stack.
+
+## License
+
+Copyright (C) 2013 by Zete Lui (BSD)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN

data/spec/zscan_spec.rb

@@ -0,0 +1,67 @@
+require_relative "../lib/zscan"
+require 'rspec/autorun'
+RSpec.configure do |config|
+  config.expect_with :stdlib
+end
+
+describe ZScan do
+  before :each do
+    @z = ZScan.new 'ab你好'
+  end
+
+  before :all do
+    GC.stress = true
+  end
+
+  it "random workflow" do
+    assert_equal 2, @z.match_bytesize('ab')
+    @z.pos = 4
+    assert_equal 8, @z.bytepos
+    @z.push
+    assert_equal nil, @z.scan(/ab你/)
+    @z.pos = 0
+    assert_equal 'ab你', @z.scan(/ab你/)
+
+    @z.restore
+    assert_equal 8, @z.bytepos
+    @z.pos = 3
+    @z.restore
+    assert_equal 8, @z.bytepos
+  end
+
+  it "scans from middle" do
+    @z.bytepos = 2
+    assert_equal '你', @z.scan('你')
+    assert_equal '好', @z.rest
+  end
+
+  it "won't overflow pos" do
+    @z.pos = 20
+    assert_equal 8, @z.bytepos
+    assert_equal 4, @z.pos
+
+    @z.skip('ab')
+    assert_equal 8, @z.bytepos
+
+    @z.pos = -1
+    assert_equal 0, @z.bytepos
+    assert_equal 0, @z.pos
+
+    @z.bytepos = 20
+    assert_equal 8, @z.bytepos
+    assert_equal 4, @z.pos
+
+    @z.bytepos = -1
+    assert_equal 0, @z.bytepos
+    assert_equal 0, @z.pos
+  end
+
+  it "recognizes anchors" do
+    z = ZScan.new "a x:b+ $ \\k<x>"
+    z.pos = 1
+    assert_equal ' ', z.scan(/\s*(\#.*$\s*)*/)
+    z.pos = 1
+    assert_equal '', z.scan(/(?<=a)/)
+    assert_equal nil, z.scan(/^/)
+  end
+end

data/zscan.gemspec

@@ -1,14 +1,15 @@
 Gem::Specification.new do |s|
   s.name = "zscan"
-  s.version = "0.5"
+  s.version = "1.0" # version mapped from zscan.rb, don't change here
   s.author = "Zete Lui"
   s.homepage = "https://github.com/luikore/zscan"
   s.platform = Gem::Platform::RUBY
   s.summary = "improved string scanner"
-  s.description = "improved string scanner"
+  s.description = "improved string scanner, respects anchors and lookbehinds, supports codepoint positioning"
   s.required_ruby_version = ">=1.9.2"
+  s.licenses = ['BSD']
 
-  s.files = %w"readme.md lib/zscan.rb ext/zscan.c ext/extconf.rb zscan.gemspec"
+  s.files = Dir.glob('{rakefile,zscan.gemspec,readme.md,**/*.{rb,c}}')
   s.require_paths = ["lib"]
   s.extensions = ["ext/extconf.rb"]
   s.rubygems_version = '1.8.24'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: zscan
 version: !ruby/object:Gem::Version
-  version: '0.5'
+  version: '1.0'
 platform: ruby
 authors:
 - Zete Lui
@@ -10,20 +10,24 @@ bindir: bin
 cert_chain: []
 date: 2013-05-06 00:00:00.000000000 Z
 dependencies: []
-description: improved string scanner
+description: improved string scanner, respects anchors and lookbehinds, supports codepoint
+  positioning
 email: 
 executables: []
 extensions:
 - ext/extconf.rb
 extra_rdoc_files: []
 files:
+- rakefile
+- zscan.gemspec
 - readme.md
+- ext/extconf.rb
 - lib/zscan.rb
+- spec/zscan_spec.rb
 - ext/zscan.c
-- ext/extconf.rb
-- zscan.gemspec
 homepage: https://github.com/luikore/zscan
-licenses: []
+licenses:
+- BSD
 metadata: {}
 post_install_message: 
 rdoc_options: []