parser 2.0.0.beta6 → 2.0.0.beta7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ff9e3b7f7bacab0b950939c52f7df525def64ecc
-  data.tar.gz: 79f7bfdc451009618f190696b9d01ab9fb2a66d4
+  metadata.gz: 2ddc24b45d61fd0c8dbd0796770c2932bdd6b6e2
+  data.tar.gz: 25b8bbe0798e066f1fa005a88e5ea3e8cdf95a20
 SHA512:
-  metadata.gz: 20a42c7ac8a381a0eff6cdd2cca9a9dc74e1b8eea8463e8b12fa910c97e5d666402db364cd9cdd1d57c7ddb17ef148dc38d171a6be5d078f62acaa78a777ba21
-  data.tar.gz: 975bf3b4ebe25e4d34e2a6b5a28cd4447ec91b1b4f5b29bed0b025a29e5a5a42b15c72d73b0b54aeeaffabdcc0357d8edd8091ce4093f8b40be616a0851a317e
+  metadata.gz: 3454dc2eb47f0555bb29c4cafe6ef65ecf18003b2dfc4406e00190652af8098f2a48160d1225ad36d655755a537de3115ccad033594bf0b946fe65687caa73ca
+  data.tar.gz: 05e32b3d2f67f0ff2c7d745efc63336f81ae83f22841f4cddb8cfe76e8a311886eeefe53eaed814c17655b05422899590afa141dca73a3e08e1daaa90f2dc0c6

data/.rubocop.yml

@@ -8,3 +8,18 @@ AllCops:
 
 HashSyntax:
   Enabled: false
+
+LineLength:
+  Max: 120
+
+MethodLength:
+  Enabled: false
+
+SymbolName:
+  Enabled: false
+
+IfUnlessModifier:
+  Enabled: false
+
+Lambda:
+  Enabled: false

data/CHANGELOG.md

@@ -1,6 +1,18 @@
 Changelog
 =========
 
+v2.0.0.beta7 (2013-06-22)
+-------------------------
+
+API modifications:
+ * Implement a much more sane encoding model (closes #60). (Peter Zotov)
+
+Features implemented:
+ * builders/default: (while-post) and (until-post); (kwbegin) (fixes #70). (Peter Zotov)
+
+Bugs fixed:
+ * builders/default: don't swallow (begin) in "if (foo); end" (fixes #75). (Peter Zotov)
+
 v2.0.0.beta6 (2013-06-17)
 -------------------------
 

data/README.md

@@ -133,7 +133,7 @@ The Bison parser rules are derived from [Ruby MRI](http://github.com/ruby/ruby)
 
 ## Contributing
 
-1. Make sure you have [Ragel 6.8](http://www.complang.org/ragel/) installed
+1. Make sure you have [Ragel ~> 6.7](http://www.complang.org/ragel/) installed
 2. Fork it
 3. Create your feature branch (`git checkout -b my-new-feature`)
 4. Commit your changes (`git commit -am 'Add some feature'`)

data/Rakefile

@@ -23,6 +23,7 @@ CLEAN.include(GENERATED_FILES)
 
 desc 'Generate the Ragel lexer and Bison parser.'
 task :generate => GENERATED_FILES do
+  Rake::Task[:ragel_check].invoke
   GENERATED_FILES.each do |filename|
     content = File.read(filename)
     content = "# -*- encoding:utf-8; warn-indent:false -*-\n" + content
@@ -42,6 +43,20 @@ task :clean_env do
   ENV.delete 'RACC_DEBUG'
 end
 
+task :ragel_check do
+  major_req, minor_req = 6, 7
+
+  ragel_check = `which ragel && ragel --version`
+  ragel_version = ragel_check.match(/version (([0-9]+)\.([0-9]+))/)
+  raise 'command-line dependency ragel not installed!' unless ragel_version
+
+  _, version_str, major, minor = *ragel_version
+  if (major.to_i != major_req || minor.to_i < minor_req) # ~> major.minor
+    raise "command-line dependency ragel must be " +
+          "~> #{major_req}.#{minor_req}; got #{version_str}"
+  end
+end
+
 desc 'Generate YARD documentation'
 task :yard => :generate do
   sh('yard doc')

data/lib/parser.rb

@@ -57,61 +57,61 @@ module Parser
 
   ERRORS = {
     # Lexer errors
-    :unicode_point_too_large => "invalid Unicode codepoint (too large)",
-    :invalid_escape          => "invalid escape character syntax",
-    :invalid_escape_use      => "invalid character syntax; use ?%{escape}",
-    :incomplete_escape       => "incomplete character syntax",
-    :invalid_hex_escape      => "invalid hex escape",
-    :invalid_unicode_escape  => "invalid Unicode escape",
-    :unterminated_unicode    => "unterminated Unicode escape",
-    :escape_eof              => "escape sequence meets end of file",
-    :string_eof              => "unterminated string meets end of file",
-    :regexp_options          => "unknown regexp options: %{options}",
+    :unicode_point_too_large => 'invalid Unicode codepoint (too large)',
+    :invalid_escape          => 'invalid escape character syntax',
+    :invalid_escape_use      => 'invalid character syntax; use ?%{escape}',
+    :incomplete_escape       => 'incomplete character syntax',
+    :invalid_hex_escape      => 'invalid hex escape',
+    :invalid_unicode_escape  => 'invalid Unicode escape',
+    :unterminated_unicode    => 'unterminated Unicode escape',
+    :escape_eof              => 'escape sequence meets end of file',
+    :string_eof              => 'unterminated string meets end of file',
+    :regexp_options          => 'unknown regexp options: %{options}',
     :cvar_name               => "`%{name}' is not allowed as a class variable name",
     :ivar_name               => "`%{name}' is not allowed as an instance variable name",
     :trailing_underscore     => "trailing `_' in number",
-    :empty_numeric           => "numeric literal without digits",
-    :invalid_octal           => "invalid octal digit",
-    :no_dot_digit_literal    => "no .<digit> floating literal anymore; put 0 before dot",
-    :bare_backslash          => "bare backslash only allowed before newline",
+    :empty_numeric           => 'numeric literal without digits',
+    :invalid_octal           => 'invalid octal digit',
+    :no_dot_digit_literal    => 'no .<digit> floating literal anymore; put 0 before dot',
+    :bare_backslash          => 'bare backslash only allowed before newline',
     :unexpected              => "unexpected `%{character}'",
-    :embedded_document       => "embedded document meats end of file (and they embark on a romantic journey)",
+    :embedded_document       => 'embedded document meats end of file (and they embark on a romantic journey)',
 
     # Lexer warnings
-    :ambiguous_literal       => "ambiguous first argument; parenthesize arguments or add whitespace to the right",
+    :ambiguous_literal       => 'ambiguous first argument; parenthesize arguments or add whitespace to the right',
     :ambiguous_prefix        => "`%{prefix}' interpreted as argument prefix",
 
     # Parser errors
-    :nth_ref_alias           => "cannot define an alias for a back-reference variable",
-    :begin_in_method         => "BEGIN in method",
-    :backref_assignment      => "cannot assign to a back-reference variable",
-    :invalid_assignment      => "cannot assign to a keyword",
-    :module_name_const       => "class or module name must be a constant literal",
-    :unexpected_token        => "unexpected token %{token}",
-    :argument_const          => "formal argument cannot be a constant",
-    :argument_ivar           => "formal argument cannot be an instance variable",
-    :argument_gvar           => "formal argument cannot be a global variable",
-    :argument_cvar           => "formal argument cannot be a class variable",
-    :duplicate_argument      => "duplicate argument name",
-    :empty_symbol            => "empty symbol literal",
-    :odd_hash                => "odd number of entries for a hash",
-    :singleton_literal       => "cannot define a singleton method for a literal",
-    :dynamic_const           => "dynamic constant assignment",
-    :module_in_def           => "module definition in method body",
-    :class_in_def            => "class definition in method body",
-    :unexpected_percent_str  => "%{type}: unknown type of percent-literal",
-    :block_and_blockarg      => "both block argument and literal block are passed",
-    :masgn_as_condition      => "multiple assignment in conditional context",
+    :nth_ref_alias           => 'cannot define an alias for a back-reference variable',
+    :begin_in_method         => 'BEGIN in method',
+    :backref_assignment      => 'cannot assign to a back-reference variable',
+    :invalid_assignment      => 'cannot assign to a keyword',
+    :module_name_const       => 'class or module name must be a constant literal',
+    :unexpected_token        => 'unexpected token %{token}',
+    :argument_const          => 'formal argument cannot be a constant',
+    :argument_ivar           => 'formal argument cannot be an instance variable',
+    :argument_gvar           => 'formal argument cannot be a global variable',
+    :argument_cvar           => 'formal argument cannot be a class variable',
+    :duplicate_argument      => 'duplicate argument name',
+    :empty_symbol            => 'empty symbol literal',
+    :odd_hash                => 'odd number of entries for a hash',
+    :singleton_literal       => 'cannot define a singleton method for a literal',
+    :dynamic_const           => 'dynamic constant assignment',
+    :module_in_def           => 'module definition in method body',
+    :class_in_def            => 'class definition in method body',
+    :unexpected_percent_str  => '%{type}: unknown type of percent-literal',
+    :block_and_blockarg      => 'both block argument and literal block are passed',
+    :masgn_as_condition      => 'multiple assignment in conditional context',
 
     # Parser warnings
-    :end_in_method           => "END in method; use at_exit",
+    :end_in_method           => 'END in method; use at_exit',
     :space_before_lparen     => "don't put space before argument parentheses",
-    :useless_else            => "else without rescue is useless",
+    :useless_else            => 'else without rescue is useless',
   }.freeze
 
   def self.check_for_encoding_support
     unless defined?(Encoding)
-      raise RuntimeError, "Parsing 1.9 and later versions of Ruby is not supported on 1.8 due to the lack of Encoding support"
+      raise RuntimeError, 'Parsing 1.9 and later versions of Ruby is not supported on 1.8 due to the lack of Encoding support'
     end
   end
 end

data/lib/parser/base.rb

@@ -12,6 +12,8 @@ module Parser
         $stderr.puts(diagnostic.render)
       end
 
+      string = string.dup.force_encoding(parser.default_encoding)
+
       source_buffer = Source::Buffer.new(file, line)
       source_buffer.source = string
 
@@ -106,11 +108,6 @@ module Parser
       @lexer.advance
     end
 
-    def value_expr(v)
-      #p 'value_expr', v
-      v
-    end
-
     def check_kwarg_name(name_t)
       case name_t[0]
       when /^[a-z_]/

data/lib/parser/builders/default.rb

@@ -718,6 +718,10 @@ module Parser
     end
 
     def loop_mod(type, body, keyword_t, cond)
+      if body.type == :kwbegin
+        type = :"#{type}_post"
+      end
+
       n(type, [ check_condition(cond), body ],
         keyword_mod_map(body, keyword_t, cond))
     end
@@ -802,7 +806,7 @@ module Parser
 
     def begin(begin_t, body, end_t)
       if body.nil?
-        # A nil expression: `()' or `begin end'.
+        # A nil expression: `()'.
         n0(:begin,
           collection_map(begin_t, nil, end_t))
       elsif body.type == :mlhs  ||
@@ -818,7 +822,21 @@ module Parser
       end
     end
 
-    alias :begin_keyword :begin
+    def begin_keyword(begin_t, body, end_t)
+      if body.nil?
+        # A nil expression: `begin end'.
+        n0(:kwbegin,
+          collection_map(begin_t, nil, end_t))
+      elsif (body.type == :begin &&
+             body.loc.begin.nil? && body.loc.end.nil?)
+        # Synthesized (begin) from compstmt "a; b".
+        n(:kwbegin, body.children,
+          collection_map(begin_t, body.children, end_t))
+      else
+        n(:kwbegin, [ body ],
+          collection_map(begin_t, [ body ], end_t))
+      end
+    end
 
     private
 
@@ -834,7 +852,9 @@ module Parser
 
       when :begin
         if cond.children.count == 1
-          check_condition(cond.children.last)
+          cond.updated(nil, [
+            check_condition(cond.children.last)
+          ])
         else
           cond
         end
@@ -843,8 +863,8 @@ module Parser
         lhs, rhs = *cond
 
         type = case cond.type
-        when :irange; :iflipflop
-        when :erange; :eflipflop
+        when :irange then :iflipflop
+        when :erange then :eflipflop
         end
 
         cond.updated(type, [

data/lib/parser/lexer/explanation.rb

@@ -30,7 +30,7 @@ module Parser
       line[from...to] = "\e[4m#{line[from...to]}\e[0m"
 
       tail_len   = to - from - 1
-      tail       = "~" * (tail_len >= 0 ? tail_len : 0)
+      tail       = '~' * (tail_len >= 0 ? tail_len : 0)
       decoration =  "#{" " * from}\e[1;31m^#{tail}\e[0m #{token} ".
                         ljust(70) + info
 

data/lib/parser/ruby18.y

@@ -236,9 +236,6 @@ rule
                 | arg
 
       expr_value: expr
-                    {
-                      result = value_expr(val[0])
-                    }
 
     command_call: command
                 | block_command
@@ -748,9 +745,6 @@ rule
                 | primary
 
        arg_value: arg
-                    {
-                      result = value_expr(val[0])
-                    }
 
        aref_args: none
                 | command opt_nl
@@ -1245,9 +1239,6 @@ rule
                     }
 
    primary_value: primary
-                    {
-                      result = value_expr(val[0])
-                    }
 
             then: term
                 | tCOLON
@@ -1942,3 +1933,7 @@ require 'parser'
   def version
     18
   end
+
+  def default_encoding
+    Encoding::BINARY
+  end

data/lib/parser/ruby19.y

@@ -267,9 +267,6 @@ rule
                 | arg
 
       expr_value: expr
-                    {
-                      result = value_expr(val[0])
-                    }
 
     command_call: command
                 | block_command
@@ -820,9 +817,6 @@ rule
                 | primary
 
        arg_value: arg
-                    {
-                      result = value_expr(val[0])
-                    }
 
        aref_args: none
                 | args trailer
@@ -1202,9 +1196,6 @@ rule
                     }
 
    primary_value: primary
-                    {
-                      result = value_expr(val[0])
-                    }
 
             then: term
                 | kTHEN
@@ -2142,3 +2133,7 @@ Parser.check_for_encoding_support
   def version
     19
   end
+
+  def default_encoding
+    Encoding::BINARY
+  end

data/lib/parser/ruby20.y

@@ -278,9 +278,6 @@ rule
                 | arg
 
       expr_value: expr
-                    {
-                      result = value_expr(val[0])
-                    }
 
     command_call: command
                 | block_command
@@ -840,9 +837,6 @@ rule
                 | primary
 
        arg_value: arg
-                    {
-                      result = value_expr(val[0])
-                    }
 
        aref_args: none
                 | args trailer
@@ -1239,9 +1233,6 @@ rule
                     }
 
    primary_value: primary
-                    {
-                      result = value_expr(val[0])
-                    }
 
             then: term
                 | kTHEN
@@ -2327,3 +2318,7 @@ Parser.check_for_encoding_support
   def version
     20
   end
+
+  def default_encoding
+    Encoding::UTF_8
+  end

data/lib/parser/ruby21.y

@@ -273,9 +273,6 @@ rule
                 | arg
 
       expr_value: expr
-                    {
-                      result = value_expr(val[0])
-                    }
 
     command_call: command
                 | block_command
@@ -835,9 +832,6 @@ rule
                 | primary
 
        arg_value: arg
-                    {
-                      result = value_expr(val[0])
-                    }
 
        aref_args: none
                 | args trailer
@@ -1240,9 +1234,6 @@ rule
                     }
 
    primary_value: primary
-                    {
-                      result = value_expr(val[0])
-                    }
 
             then: term
                 | kTHEN
@@ -2347,3 +2338,7 @@ warn "warning: Ruby 2.1 is not released yet and parser support may be incomplete
   def version
     21
   end
+
+  def default_encoding
+    Encoding::BINARY
+  end

data/lib/parser/runner.rb

@@ -102,7 +102,7 @@ module Parser
       end
 
       if @files.empty? && @fragments.empty?
-        $stderr.puts "Need something to parse!"
+        $stderr.puts 'Need something to parse!'
         exit 1
       end
 
@@ -141,6 +141,8 @@ module Parser
 
     def process_fragments
       @fragments.each_with_index do |fragment, index|
+        fragment = fragment.dup.force_encoding(@parser.default_encoding)
+
         buffer = Source::Buffer.new("(fragment:#{index})")
         buffer.source = fragment
 
@@ -150,8 +152,11 @@ module Parser
 
     def process_files
       @files.each do |filename|
+        source = File.read(filename)
+        source.force_encoding(@parser.default_encoding)
+
         buffer = Parser::Source::Buffer.new(filename)
-        buffer.read
+        buffer.source = source
 
         process_buffer(buffer)
       end