parser 2.0.0.pre7 → 2.0.0.pre8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 776614b2c048d80ca7eadd5fd3c3f914c51d88ec
-  data.tar.gz: 01ddc7d4877aa4198de7685707d410febc010c29
+  metadata.gz: 784e6ea2b211d0e66b2c9b557ac575c2f7204444
+  data.tar.gz: 381fae65cbb0abd8400e382c8bf926221f74929f
 SHA512:
-  metadata.gz: bab2b3791e25cfe5a13fc6cd844375377c3d678f302b79f3580d10fad2cb6dda60d5d371c8388142899eb803a238c84c9ccc2d2d46d3d2d00f80b53c08df7afc
-  data.tar.gz: be57c964d5423e2b7f53a39db34a7e668f31fe5f9fe032678e1f4d83030f304ba3150f345c6c4b4ccfa64ffde2c652df9adc179febc7d2bfa0a8a2cb3e464eaa
+  metadata.gz: 720ec8f90cb16e1e38ddd02006afe4dda5ca2c9ccf31087a0906cafd406ba6865e57d5545be81f64dd17cc710c33d108e73476742475264147049d034c97c6c4
+  data.tar.gz: f0b89adbfa47c7ba825c9d54df1d32cb0c81dec28866caea61baebd77290eb860317f3e357c10f4953056b9b1339085d0b9f2bd418848de46053b961044479c0

data/CHANGELOG.md

@@ -1,6 +1,12 @@
 Changelog
 =========
 
+v2.0.0.pre8 (2013-09-15)
+------------------------
+
+API modifications:
+ * lexer.rl: make lexing faster and improve parsing speed by ~60%. (Peter Zotov)
+
 v2.0.0.pre7 (2013-09-10)
 ------------------------
 

data/README.md

@@ -15,12 +15,7 @@ Sponsored by [Evil Martians](http://evilmartians.com).
 
 ## Installation
 
-Most recent version of Parser is 2.0; however, per
-[release schedule](https://github.com/whitequark/parser/issues/51), it stays in
-the beta status for a while. However, it handles much more input than stable
-1.x branch, and for new work it is advisable to use the beta versions.
-
-    $ gem install parser --pre
+    $ gem install parser -v=2.0
 
 ## Usage
 

data/Rakefile

@@ -114,7 +114,7 @@ task :changelog do
 end
 
 rule '.rb' => '.rl' do |t|
-  sh "ragel -R #{t.source} -o #{t.name}"
+  sh "ragel -F1 -R #{t.source} -o #{t.name}"
 end
 
 rule '.rb' => '.y' do |t|

data/lib/parser/lexer.rl

@@ -157,7 +157,17 @@ class Parser::Lexer
     @source_buffer = source_buffer
 
     if @source_buffer
-      @source = @source_buffer.source + "\0"
+      @source = @source_buffer.source
+
+      if defined?(Encoding)
+        @encoding   = @source.encoding
+
+        # This is a workaround for 1.9.2, which (without force_encoding)
+        # would convert the result to UTF-8 (source encoding of lexer.rl).
+        @source    += "\0".force_encoding(@encoding)
+      else
+        @source    += "\0"
+      end
 
       if defined?(Encoding) && @source.encoding == Encoding::UTF_8
         @source_pts = @source.unpack('U*')
@@ -165,10 +175,6 @@ class Parser::Lexer
         @source_pts = @source.unpack('C*')
       end
 
-      if defined?(Encoding)
-        @encoding = @source.encoding
-      end
-
       if @source_pts.size > 1_000_000 && @source.respond_to?(:encode)
         # A heuristic: if the buffer is larger than 1M, then
         # store it in UTF-32 and convert the tokens as they're
@@ -229,11 +235,8 @@ class Parser::Lexer
 
     # Ugly, but dependent on Ragel output. Consider refactoring it somehow.
     _lex_trans_keys         = self.class.send :_lex_trans_keys
-    _lex_actions            = self.class.send :_lex_actions
-    _lex_key_offsets        = self.class.send :_lex_key_offsets
+    _lex_key_spans          = self.class.send :_lex_key_spans
     _lex_index_offsets      = self.class.send :_lex_index_offsets
-    _lex_single_lengths     = self.class.send :_lex_single_lengths
-    _lex_range_lengths      = self.class.send :_lex_range_lengths
     _lex_indicies           = self.class.send :_lex_indicies
     _lex_trans_targs        = self.class.send :_lex_trans_targs
     _lex_trans_actions      = self.class.send :_lex_trans_actions

data/lib/parser/lexer/literal.rb

@@ -1,4 +1,4 @@
-# encoding: utf-8
+# encoding: binary
 
 module Parser
 
@@ -40,8 +40,8 @@ module Parser
 
       # DELIMITERS and TYPES are hashes with keys encoded in binary.
       # Coerce incoming data to the same encoding.
-      str_type  = coerce_encoding(str_type)
-      delimiter = coerce_encoding(delimiter)
+      str_type     = coerce_encoding(str_type)
+      delimiter    = coerce_encoding(delimiter)
 
       unless TYPES.include?(str_type)
         message = ERRORS[:unexpected_percent_str] % { :type => str_type }
@@ -204,8 +204,7 @@ module Parser
 
     def coerce_encoding(string)
       if defined?(Encoding)
-        string.encode(Encoding::UTF_8,
-                      :invalid => :replace, :undef => :replace)
+        string.dup.force_encoding(Encoding::BINARY)
       else
         string
       end

data/lib/parser/version.rb

@@ -1,3 +1,3 @@
 module Parser
-  VERSION = '2.0.0.pre7'
+  VERSION = '2.0.0.pre8'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parser
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre7
+  version: 2.0.0.pre8
 platform: ruby
 authors:
 - Peter Zotov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-09 00:00:00.000000000 Z
+date: 2013-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ast