parser 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
---- 
-SHA512: 
-  metadata.gz: 9770aa1a6462f5becb021dd1273bb992ce984aa07f4d2ab3184f909b4c9d5b7bf207c62e1c2d1f6684dfbde36c2e12cad77824d9b2343c86569b8d37d8b40259
-  data.tar.gz: c84a8295a37621c885de2618f8fc514469c239b564ca29ca75a576e0a1133f396707382aeccc1b4d73347671b081c0162b84bed32546796ea1fae02d66968aad
-SHA1: 
-  metadata.gz: 4a29479081e4085c756ccbfe0841c4b8244ce7ac
-  data.tar.gz: b2bdbd92ecf63102deafdfe71be38930d2c33930
+---
+SHA1:
+  metadata.gz: 16e6e6480aa89b565b60ceca67280519331836c0
+  data.tar.gz: 023823bcfbdf9f66858ed4de49d738cd94587891
+SHA512:
+  metadata.gz: 999557fa55073dbd6077d224409d740b6ec68b9f0198906db4e5bd20276adfb3eab75aa9e2823369f5db8bcf474c7e37d709354c9bf8ffdd1200ffe7ddcdaa45
+  data.tar.gz: af1727c35cd4bdb1aea2735c9613c517ad363e9f3511a78d5287802b80fcfdbcf81bf6ce875d61e5f4e0fe8ff793adfde7692e882ac877519f51c7c53a177bf1

data/.gitignore

@@ -7,7 +7,6 @@ Gemfile.lock
 InstalledFiles
 _yardoc
 coverage
-doc/
 lib/bundler/man
 pkg
 rdoc

data/README.md

@@ -41,10 +41,12 @@ p parser.parse(buffer)
 #     (send nil :bar)))
 ```
 
+If you reuse the same parser object for multiple `#parse` runs, you need to `#reset` it.
+
 ## Features
 
- * Precise source location reporting (WIP, no, not really yet).
- * [Documented](AST_FORMAT.md) AST format which is convenient to work with.
+ * Precise source location reporting.
+ * [Documented](doc/AST_FORMAT.md) AST format which is convenient to work with.
  * A simple interface and a powerful, tweakable one.
  * Parses 1.8, 1.9, 2.0 and 2.1 (preliminary) syntax with backwards-compatible AST formats.
  * Parsing error recovery.

data/bin/{parse → ruby-parse}

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
 $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
-require 'parser/all'
+require 'parser/runner/ruby_parse'
 
-p Parser::Ruby20.parse(ARGF.read)
+Parser::Runner::RubyParse.go(ARGV)

data/bin/ruby-rewrite

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
+require 'parser/runner/ruby_rewrite'
+
+Parser::Runner::RubyRewrite.go(ARGV)

data/{AST_FORMAT.md → doc/AST_FORMAT.md}

@@ -296,6 +296,7 @@ Format:
 (const (cbase) :Foo)
 "::Foo"
    ~~~ name
+ ~~ double_colon
  ~~~~~ expression
 ```
 
@@ -306,6 +307,7 @@ Format:
 (const (lvar :a) :Foo)
 "a::Foo"
     ~~~ name
+  ~~ double_colon
  ~~~~~~ expression
 ```
 
@@ -590,7 +592,7 @@ s(:op_asgn1, s(:ivar, :@foo), s(:arglist, s(:int, 0)), :"||", s(:int, 1))
 
 Format:
 ```
-(module (const nil :Foo) (begin))
+(module (const nil :Foo) (nil))
 "module Foo; end"
  ~~~~~~ keyword
              ~~~ end
@@ -600,25 +602,29 @@ Format:
 
 Format:
 ```
-(class (const nil :Foo) (const nil :Bar) (begin))
+(class (const nil :Foo) (const nil :Bar) (nil))
 "class Foo < Bar; end"
  ~~~~~ keyword    ~~~ end
+           ~ operator
+ ~~~~~~~~~~~~~~~~~~~~ expression
 
-(class (const nil :Foo) nil (begin))
+(class (const nil :Foo) nil (nil))
 "class Foo; end"
  ~~~~~ keyword
             ~~~ end
+ ~~~~~~~~~~~~~~ expression
 ```
 
 ### Singleton class
 
 Format:
 ```
-(sclass (lvar :a) (begin))
+(sclass (lvar :a) (nil))
 "class << a; end"
  ~~~~~ keyword
        ~~ operator
              ~~~ end
+ ~~~~~~~~~~~~~~~ expression
 ```
 
 ## Method (un)definition
@@ -834,6 +840,8 @@ Format:
 (send nil :foo (lvar :bar))
 "foo(bar)"
  ~~~ selector
+    ^ begin
+        ^ end
  ~~~~~~~~ expression
 ```
 
@@ -844,6 +852,8 @@ Format:
 (send (lvar :foo) :bar (int 1))
 "foo.bar(1)"
      ~~~ selector
+        ^ begin
+          ^ end
  ~~~~~~~~~~ expression
 
 (send (lvar :foo) :+ (int 1))
@@ -858,22 +868,18 @@ Format:
 
 (send (lvar :foo) :a= (int 1))
 "foo.a = 1"
-     ~~~ selector
+     ~ selector
        ^ operator
  ~~~~~~~~~ expression
 
 (send (lvar :foo) :[] (int 1))
 "foo[i]"
     ~~~ selector
-    ^ begin
-      ^ end
  ~~~~~~ expression
 
 (send (lvar :bar) :[]= (int 1) (int 2) (lvar :baz))
 "bar[1, 2] = baz"
-    ~~~~~~~~ selector
-    ^ begin
-        ^ end
+    ~~~~~~ selector
            ^ operator
  ~~~~~~~~~~~~~~~ expression
 
@@ -1066,9 +1072,10 @@ Format:
 Format:
 ```
 (when (regexp "foo" (regopt)) (begin (lvar :bar)))
-"when /foo/; bar"
+"when /foo/ then bar"
  ~~~~ keyword
- ~~~~~~~~~~ expression
+            ~~~~ begin
+ ~~~~~~~~~~~~~~~~~~~ expression
 
 (when (int 1) (int 2) (send nil :meth))
 "when 1, 2; meth"
@@ -1242,8 +1249,6 @@ Format:
 (return (lvar :foo))
 "return(foo)"
  ~~~~~~ keyword
-       ^ begin
-           ^ end
  ~~~~~~~~~~~ expression
 ```
 
@@ -1255,20 +1260,21 @@ Format:
 ```
 (resbody (array (const nil :Exception) (const nil :A)) (lvasgn :bar) (int 1))
 "rescue Exception, A => bar; 1"
- ~~~~~~ keyword      ~~ operator
+ ~~~~~~ keyword      ~~ assoc
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ expression
 
 "rescue Exception, A => bar then 1"
- ~~~~~~ keyword      ~~ operator
+ ~~~~~~ keyword      ~~ assoc
                             ~~~~ begin
 
 (resbody (array (const nil :Exception)) (ivasgn :bar) (int 1))
 "rescue Exception => @bar; 1"
- ~~~~~~ keyword   ~~ operator
+ ~~~~~~ keyword   ~~ assoc
 
 (resbody nil (lvasgn :bar) (int 1))
 "rescue => bar; 1"
  ~~~~~~ keyword
-        ~~ operator
+        ~~ assoc
 
 (resbody nil nil (int 1))
 "rescue; 1"
@@ -1281,27 +1287,34 @@ Format:
 
 Format:
 ```
-(rescue (send nil :foo) (resbody ...) (resbody ...) nil)
+(begin
+  (rescue (send nil :foo) (resbody ...) (resbody ...) nil))
 "begin; foo; rescue Exception; rescue; end"
  ~~~~~ begin                           ~~~ end
+             ~~~~~~~~~~~~~~~~~ expression (rescue.resbody/1)
+                               ~~~~~~~ expression (rescue.resbody/2)
+        ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ expression (rescue)
 ```
 
 ##### With else
 
 Format:
 ```
-(rescue (send nil :foo) (resbody ...) (resbody ...) (true))
+(begin
+  (rescue (send nil :foo) (resbody ...) (resbody ...) (true)))
 "begin; foo; rescue Exception; rescue; else true end"
- ~~~~~ begin                           ~~~~ else ~~~ end
+ ~~~~~ begin                           ~~~~ else (rescue)
+                                                 ~~~ end
 ```
 
 #### Ensure statement
 
 Format:
 ```
-(ensure (send nil :foo) (send nil :bar))
+(begin
+  (ensure (send nil :foo) (send nil :bar))
 "begin; foo; ensure; bar; end"
- ~~~~~ begin ~~~~~~ keyword
+ ~~~~~ begin ~~~~~~ keyword (ensure)
                           ~~~ end
 ```
 
@@ -1309,14 +1322,17 @@ Format:
 
 Format:
 ```
-(ensure (rescue (send nil :foo) (resbody ...) (int 1)) (send nil :bar))
+(begin
+  (ensure
+    (rescue (send nil :foo) (resbody ...) (int 1))
+    (send nil :bar))
 "begin; foo; rescue; nil; else; 1; ensure; bar; end"
- ~~~~~ begin (rescue)
- ~~~~~ begin (ensure)
-                          ~~~~ else (rescue)
+ ~~~~~ begin
+                          ~~~~ else (ensure.rescue)
+             ~~~~~~~~~~~~~~~~~~~~~ expression (rescue)
                                    ~~~~~~ keyword (ensure)
-                                                ~~~ end (rescue)
-                                                ~~~ end (ensure)
+             ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ expression (ensure)
+                                                ~~~ end
 ```
 
 #### Retry

data/doc/CUSTOMIZATION.md

@@ -0,0 +1,37 @@
+# Customizing Parsers
+
+While the default setup of the parsers provided by this Gem should be suitable
+for most some developers might want to change parts of it. An example would be
+the use of a custom class for nodes instead of `Parser::AST::Node`.
+
+Customizing the AST is done by creating a custom builder class and passing it
+to the constructor method of a parser. The default setup comes down to the
+following:
+
+    builder = Parser::Builders::Default.new
+    parser  = Parser::Ruby19.new(builder)
+
+When creating your own builder class it's best to subclass the default one so
+that you don't have to redefine every used method again:
+
+    class MyBuilder < Parser::Builders::Default
+
+    end
+
+    builder = MyBuilder.new
+    parser  = Parser::Ruby19.new(builder)
+
+## Custom Node Classes
+
+To use a custom node class you have to override the method
+`Parser::Builders::Default#n`:
+
+    class MyBuilder < Parser::Builders::Default
+      def n(type, children, source_map)
+        return MyNodeClass.new(type, children, :source_map => source_map)
+      end
+    end
+
+Note that the used class (and corresponding instance) must be compatible with
+`Parser::AST::Node` so it's best to subclass it and override/add code where
+needed.

data/doc/INTERNALS.md

@@ -0,0 +1,21 @@
+Entry points
+------------
+
+Parser should be kept as slim as possible. This includes not loading
+any potentially large files when they are likely to be unused in practice.
+
+Parser has five main (classes of) `require` entry points:
+
+ * `require 'parser'`. Main entry point, requires all classes which
+   are used across the entire library.
+ * `require 'parser/rubyXX'`. Version-specific entry point. Can raise
+   a NotImplementedError if current Ruby runtime is unable to parse the
+   requested Ruby version.
+ * `require 'parser/all'`. Requires all available parsers for released
+   versions of Ruby. Can raise NotImplementedError.
+ * `require 'parser/runner'`. Requires all the stuff which is useful for
+   command-line tools but not otherwise.
+ * `require 'parser/runner/X'`. Runner-specific entry point.
+
+All non-main entry points internally `require 'parser'`. Additionally, all
+runner-specific entry points internally `requre 'parser/runner'`.

data/lib/parser.rb

@@ -9,6 +9,8 @@ end
 
 # Library namespace
 module Parser
+  require 'parser/version'
+
   require 'parser/ast/node'
   require 'parser/ast/processor'
 
@@ -17,7 +19,17 @@ module Parser
 
   require 'parser/source/map'
   require 'parser/source/map/operator'
-  require 'parser/source/map/variable_assignment'
+  require 'parser/source/map/collection'
+  require 'parser/source/map/constant'
+  require 'parser/source/map/variable'
+  require 'parser/source/map/keyword'
+  require 'parser/source/map/definition'
+  require 'parser/source/map/send'
+  require 'parser/source/map/block'
+  require 'parser/source/map/condition'
+  require 'parser/source/map/ternary'
+  require 'parser/source/map/for'
+  require 'parser/source/map/rescue_body'
 
   require 'parser/syntax_error'
   require 'parser/diagnostic'
@@ -28,7 +40,6 @@ module Parser
   require 'parser/lexer'
   require 'parser/lexer/literal'
   require 'parser/lexer/stack_state'
-  require 'parser/lexer/explanation'
 
   module Builders
     require 'parser/builders/default'
@@ -55,7 +66,7 @@ module Parser
     :invalid_octal           => "invalid octal digit",
     :no_dot_digit_literal    => "no .<digit> floating literal anymore; put 0 before dot",
     :bare_backslash          => "bare backslash only allowed before newline",
-    :unexpected              => "unexpected %{character}",
+    :unexpected              => "unexpected `%{character}'",
     :embedded_document       => "embedded document meats end of file (and they embark on a romantic journey)",
 
     # Lexer warnings

data/lib/parser/ast/node.rb

@@ -5,6 +5,12 @@ module Parser
       attr_reader :source_map
 
       alias src source_map
+
+      def assign_properties(properties)
+        if (source_map = properties[:source_map])
+          @source_map = source_map
+        end
+      end
     end
 
   end

data/lib/parser/ast/processor.rb

@@ -2,6 +2,222 @@ module Parser
   module AST
 
     class Processor < ::AST::Processor
+      def process_regular_node(node)
+        node.updated(nil, process_all(node))
+      end
+
+      alias on_dstr     process_regular_node
+      alias on_dsym     process_regular_node
+      alias on_regexp   process_regular_node
+      alias on_xstr     process_regular_node
+      alias on_splat    process_regular_node
+      alias on_array    process_regular_node
+      alias on_pair     process_regular_node
+      alias on_hash     process_regular_node
+      alias on_irange   process_regular_node
+      alias on_erange   process_regular_node
+
+      def on_var(node)
+        name, = *node
+
+        node.updated
+      end
+
+      def process_variable_node(node)
+        on_var(node)
+      end
+
+      alias on_lvar     process_variable_node
+      alias on_ivar     process_variable_node
+      alias on_gvar     process_variable_node
+      alias on_cvar     process_variable_node
+      alias on_back_ref process_variable_node
+      alias on_nth_ref  process_variable_node
+
+      def on_vasgn(node)
+        name, value_node = *node
+
+        value_node = process(value_node) if value_node
+        node.updated(nil, [ name, value_node ])
+      end
+
+      def process_var_asgn_node(node)
+        on_vasgn(node)
+      end
+
+      alias on_lvasgn   process_var_asgn_node
+      alias on_ivasgn   process_var_asgn_node
+      alias on_gvasgn   process_var_asgn_node
+      alias on_cvdecl   process_var_asgn_node
+      alias on_cvasgn   process_var_asgn_node
+
+      alias on_and_asgn process_regular_node
+      alias on_or_asgn  process_regular_node
+
+      def on_op_asgn(node)
+        var_node, method_name, value_node = *node
+
+        node.updated(nil, [
+          process(var_node), method_name, process(value_node)
+        ])
+      end
+
+      alias on_mlhs     process_regular_node
+      alias on_masgn    process_regular_node
+
+      def on_const(node)
+        scope_node, name = *node
+
+        scope_node = process(scope_node) if scope_node
+        node.updated(nil, [ scope_node, name ])
+      end
+
+      def on_cdecl(node)
+        scope_node, name, value_node = *node
+
+        scope_node = process(scope_node) if scope_node
+        value_node = process(value_node)
+        node.updated(nil, [ scope_node, name, value_node ])
+      end
+
+      alias on_args     process_regular_node
+
+      def on_argument(node)
+        arg_name, value_node = *node
+
+        value_node = process(value_node) if value_node
+        node.updated(nil, [ arg_name, value_node ])
+      end
+
+      def process_argument_node(node)
+        on_argument(node)
+      end
+
+      alias on_arg            process_argument_node
+      alias on_optarg         process_argument_node
+      alias on_splatarg       process_argument_node
+      alias on_blockarg       process_argument_node
+      alias on_kwarg          process_argument_node
+      alias on_kwoptarg       process_argument_node
+      alias on_kwsplatarg     process_argument_node
+
+      alias on_arg_expr       process_regular_node
+      alias on_restarg_expr   process_regular_node
+      alias on_blockarg_expr  process_regular_node
+
+      alias on_module   process_regular_node
+
+      def on_class(node)
+        name_node, superclass_node, body_node = *node
+
+        superclass_node = process(superclass_node) if superclass_node
+        node.updated(nil, [
+          name_node, superclass_node, process(body_node)
+        ])
+      end
+
+      alias on_sclass   process_regular_node
+
+      def on_def(node)
+        name, args_node, body_node = *node
+
+        node.updated(nil, [
+          name,
+          process(args_node), process(body_node)
+        ])
+      end
+
+      def on_defs(node)
+        definee_node, name, args_node, body_node = *node
+
+        node.updated(nil, [
+          process(definee_node), name,
+          process(args_node), process(body_node)
+        ])
+      end
+
+      alias on_undef    process_regular_node
+      alias on_alias    process_regular_node
+
+      def on_send(node)
+        receiver_node, method_name, *arg_nodes = *node
+
+        receiver_node = process(receiver_node) if receiver_node
+        node.updated(nil, [
+          receiver_node, method_name, *process_all(arg_nodes)
+        ])
+      end
+
+      alias on_block    process_regular_node
+
+      alias on_while    process_regular_node
+      alias on_until    process_regular_node
+      alias on_for      process_regular_node
+
+      alias on_return   process_regular_node
+      alias on_break    process_regular_node
+      alias on_next     process_regular_node
+      alias on_redo     process_regular_node
+      alias on_retry    process_regular_node
+      alias on_super    process_regular_node
+      alias on_yield    process_regular_node
+      alias on_defined? process_regular_node
+
+      alias on_not      process_regular_node
+      alias on_and      process_regular_node
+      alias on_or       process_regular_node
+
+      def on_if(node)
+        cond_node, if_true_node, if_false_node = *node
+
+        if_true_node  = process(if_true_node)  if if_true_node
+        if_false_node = process(if_false_node) if if_false_node
+
+        node.updated(nil, [
+          process(cond_node),
+          if_true_node, if_false_node
+        ])
+      end
+
+      alias on_when     process_regular_node
+
+      def on_case(node)
+        cond_node, *bodies = *node
+        when_nodes, else_node = bodies[0..-2], bodies[-1]
+
+        else_node = process(else_node) if else_node
+        node.updated(nil, [
+          process(cond_node),
+          *(process_all(when_nodes) << else_node)
+        ])
+      end
+
+      def on_resbody(node)
+        exc_list_node, exc_var_node, body_node = *node
+
+        exc_list_node = process(exc_list_node) if exc_list_node
+        exc_var_node  = process(exc_var_node)  if exc_var_node
+
+        node.updated(nil, [
+          exc_list_node, exc_var_node,
+          process(body_node)
+        ])
+      end
+
+      def on_rescue(node)
+        body_node, *handlers = *node
+        handler_nodes, else_node = handlers[0..-2], handlers[-1]
+
+        else_node = process(else_node) if else_node
+        node.updated(nil, [
+          process(body_node),
+          *(process_all(handler_nodes) << else_node)
+        ])
+      end
+
+      alias on_ensure   process_regular_node
+
+      alias on_begin    process_regular_node
     end
 
   end