redparse 0.8.1 → 0.8.2

data/lib/redparse/generate.rb

@@ -0,0 +1,378 @@
+class RedParse
+=begin
+3 LR-Parsing Mechanics
+We briefly explain the fundamentals of shift-reduce parsing (which represents 
+the LR(1) family) without going into any more detail than necessary for 
+subsequent exposition. LALR(1) parsers like yacc simulate, either directly or 
+indirectly, a very simple automaton with a stack of automaton states [FL88]. 
+(Parsers generated by yacc also maintain a semantic stack, but since that stack 
+grows in parallel with the state stack, we only describe the use of the state
+stack here.) Simulating the automaton requires two mechanisms: one for 
+determining the action, which is determined by the current input symbol and the 
+state on the top of the stack, and one for determining state transitions based 
+on the current top of stack and a grammar symbol. At parser-generation time 
+LALR(1) grammar analysis builds these tables, called action and goto, 
+respectively. (The analysis is necessary regardless of whether a table-driven 
+or hard-coded parser is desired.) Functionally, these tables have the following 
+signatures.
+
+goto: state x symbol -> state
+action: state x token -> {shift,reduce_y,accept,error}
+
+There are only four possible actions: reduce, shift, accept, and error. Reduce 
+actions are parameterized by the grammar production being reduced. Actions are 
+described below. let TOS be the state on the top of the stack, and let 
+la_identity be the current lookahead token.
+
+shift  A shift pushes goto[TOS,la_identity] onto the stack, and updates 
+la_identity by advancing the lexical analyzer.
+
+reduce_y  A reduction processes production Y : X -> x_1...x_n, which 
+requires popping n states off the stack, followed by pushing goto[TOS, X]. 
+(The semantic action of the parser relating to this production would be 
+executed prior to popping states off the stack.)
+
+accept  An accept signals a successful parse.
+
+error  An error requires error reporting and/or recovery.
+
+
+4 Simple Implementation
+mule creates a single parsing routine, yyparse(), that simulates the LALR(1) 
+parser directly in ANSI C, without interpreting any tables. The routine has 
+five simple parts: initialization, automata states, reduction actions, 
+nonterminal transitions, and error recovery. Although very similar to the
+inverted table structure in [Pfa90], this structure avoids the duplication 
+of semantic action routines. Another diverence is the yacc-compatible error 
+recovery. The structure is simple, with all code being generated from a tiny 
+set of small, well-defined templates that directly mirror the grammar or 
+LALR(1) automaton. Since both the state stack and the semantic stack grow in 
+unison, we wrap the stack entries into a single structure, StackType.
+
+4.1 Initialization
+The initalization phase simply sets up bookkeeping and data structures for 
+subsequent automata simulation. It is grammar-independent.
+=end
+
+def init_code
+"
+#define YYABORT do { \\
+  free(start_stack);return -1; \\
+  } while(0)
+#define YYACCEPT do { \\
+  YYSTYPE result=SEMANTIC_STACK; \\
+  free(start_stack); \\
+  return result; \\
+  } while(0)
+/*#define yyclearin_token = yylex(&la_token)*/
+#define yyerrok yyerrorstatus = 3
+#define YYERROR goto user_error_handler
+#define YYRECOVERING() (yyerrorstatus <= 2)
+
+
+typedef VALUE YYSTYPE;
+
+#if 0
+typedef struct stackType{
+  int state;// State stack element.
+} StackType;
+typedef struct {
+  VALUE semantic;
+} SemanticStackType;
+#else
+typedef int StackType;
+typedef VALUE SemanticStackType;
+#end
+int yyparse(void){
+  YYSTYPE la_token;// Semantic value computed by yylex().
+  int la_identity;
+  unsigned yyerrorstatus = 3;// Initialize error-recovery counter.
+  YYSTYPE yyredval;// Variable holds semantic value of$$.
+  VALUE semantic_stack; /*Array of Node|Token*/
+//  SemanticStackType *semantic_stack_start;
+  StackType *stack_start;// Stack.
+  unsigned i=0;
+  unsigned stack_size=64;
+  
+  stack_start=realloc(NULL,sizeof(StackType)*stack_size); 
+  if (stack_start==NULL) MALLOC_ERROR(); 
+  semantic_stack=rb_ary_new();
+//  semantic_stack_start=realloc(NULL,sizeof(SemanticStackType)*stack_size); 
+//  if (semantic_stack_start==NULL) MALLOC_ERROR(); 
+
+  la_identity = yylex(&la_token); /* Get 1st token.*/
+
+  goto shift_state_#{str2cname all_states.first.name};/* Start state.*/
+"
+end
+
+
+def state_utils
+"
+#define MALLOC_ERROR() huh
+#define RESERVE_STACK_SLOT() \\
+  if (++i >= stack_size){ \\
+    unsigned new_stack_size=stack_size*2; \\
+    stack_start=realloc(stack_start,sizeof(StackType)*new_stack_size); \\
+    if (stack_start==NULL) MALLOC_ERROR(); \\
+    //semantic_stack_start=realloc(semantic_stack_start,sizeof(SemanticStackType)*new_stack_size); \\
+    //if (semantic_stack_start==NULL) MALLOC_ERROR(); \\
+    stack_size=new_stack_size; \\
+  }
+
+#define GET_TOKEN() \\
+  do { \\
+    SEMANTIC_STACK_SET(la_token); /*Put lexical semantic entry on stack.*/ \\
+    la_identity = yylex(&la_token); /* Advance lexical analysis.*/ \\
+    yyerrorstatus++; /* Update error-recovery counter.*/ \\
+  } while(0)
+
+#define STACK stack_start[i]
+#define SEMANTIC_STACK rb_ary_get(semantic_stack,rb_int2fixnum(i))
+#define SEMANTIC_STACK_SET(x) rb_ary_set(semantic_stack,rb_int2fixnum(i),x)
+#define OLDSTACK stack_start[i-1]
+"
+end
+
+def action2c(action)
+             case action
+             when Rule; "goto reduce_#{str2cname action.name};"
+             when nil,:error;  "goto error_handler;"
+             when ParserState;  "goto shift_state_#{str2cname action.name};"
+             when :accept; "YYACCEPT;"
+             when MultiReduce; action.action2c
+             when MultiShift; action.action2c
+#             when StackMonkey; action.action2c
+             else fail "unexpected action type: #{action.class} = #{action}"
+             end
+end
+
+=begin
+4.2 Hard-coded States
+For each automata state, mule creates code responsible for simulating the action of that state based
+on the current input token. All transitions
+into
+a given state are labeled with the same grammar
+symbol. States labeled with a token are called
+shift states
+and they require extra code to advance
+the lexical analyzer. The template of this code for state N is
+=end
+
+def state(state_n,n)
+#n=state_n.small_int
+name=state_n.name
+"
+shift_state_#{name}: 
+  GET_TOKEN();  /*modifies token, la_token*/
+state_#{name}:  /*state_#{n}:*/
+  STACK = #{n};
+
+  RESERVE_STACK_SLOT();
+state_action_#{name}: /* Error-recovery entry point.*/
+/*state_action_#{n}:*/
+  switch (la_identity){
+    #{state_n.actions.map do |tok,action|
+        %[  case #{str2cname(tok)}: #{action2c action}]
+      end.join(%[\n])
+    }
+    default: #{action2c state_n.actions.default}
+  }
+"
+rescue Exception=>e
+  backtrace.unshift("exception in state #{name} #{e.class}:#{e}").join("\n")
+end
+
+=begin
+The state number is stored in the stack, followed by possibly invoking the lexical analyzer. The
+three optional lines store the semantic value of the current token, advance the lexical analyzer, and
+do error-recovery bookkeeping. Incrementing the stack pointer completes the push. The case arms
+of the switch are determined by the
+action
+table computed by the LALR(1) analysis; for each
+condition met in the comments, a case arm must be generated. Default actions were developed for
+compressing table-driven parsers, and can be similarly employed here for generating the switchs
+default [FL88].
+
+4.3 Reduction Actions
+One piece of code is generated for each production. Its template is given below.
+=end
+
+def repl(rule,m)
+  repl=rule.replacement
+  case repl
+  when :shift,:accept #do nothing?
+  when Class
+    %[static VALUE repl_#{rule.name}=rb_const_lookup(rb_const_lookup(kNIL,"RedParse"),"#{repl.name});\n]
+  when StackMonkey
+end
+
+def reduce(rule,m)
+  repl=rule.replacement
+  pattern=rule.pattern
+  numpatterns=pattern.subregs.size
+
+  leader="\nreduce_#{rule.name}:\n/*reduce_#{m}:*/ /*Production #{m}: #{rule.to_s} */\n"
+  if StackMonkey===repl
+    userhook="
+      huh rb_proc_call(stack_monkey_#{rule.name},semantic_stack);
+      huh /*objects removed from semantic stack get pushed back onto lexer input*/
+      i -= #{repl.backup_count};
+      
+      goto nonterminal_switcher_#{str2cname repl.huh}; 
+      // Compute transition on some old node from the (semantic) stack.
+      huh//need a node (or token?) number to goto address mapper (switch stmt) here    
+    "
+  elsif :accept==repl
+    return leader+"YYACCEPT;"
+  end
+  %[
+    #{leader}#{userhook}
+    yyredval=huh rb_call(repl_#{rule.name},huh("create"),semantic_stack,huh2ruby(-#{numpatterns}));
+
+    i -= #{numpatterns}; /* Pop patterns from stack.*/
+  
+    SEMANTIC_STACK_SET(yyredval); /* Copy ($$) onto semantic stack.*/
+    goto nonterminal_#{str2cname repl.name}; /* Compute transition on produced node type.*/
+  ]
+  end
+
+rescue Exception=>e
+  backtrace.unshift("exception in reduce of rule #{rule.name} #{e.class}:#{e}").join("\n")
+end
+
+=begin
+User actions are associated with reductions, and the code corresponding to a given production
+is expanded in-place. After the user code, the symbols associated with right-hand side of the
+production are popped, followed by copying $$ onto the semantic stack. Finally, there is a jump to
+the code that will compute the appropriate state given the left-hand side symbol of this production.
+
+4.4 Nonterminal Transitions
+For each nonterminal, code is produced to compute (and jump to) the appropriate state given the
+current state. This simple switch statement is given below.
+=end
+
+def nonterminal(j)
+"
+nonterminal_#{str2cname j.name}:  /*nonterminal_#{j.small_int}:*/
+  switch (OLDSTACK){   // Top of stack.
+    #{
+      all_states.map_with_index do|state,k|
+        %[  case #{k}: goto state_#{str2cname state.goto[j].name};\n]
+      end
+    }
+  }
+"
+rescue Exception=>e
+  backtrace.unshift("exception in node(nonterminal) #{j.name} #{e.class}:#{e}").join("\n")
+end
+
+=begin
+The case arms of the switch statement are taken directly from the goto table 
+that was computed by the LALR(1) grammar analysis. Because this switch cannot 
+fail, no default entry is needed. However, making the most common case arm the 
+default is a trivial time and space optimization.
+=end
+
+def generate_c output
+  output<< init_code
+  output<< state_utils
+  (0...RULES().size).each_with_index{|i,m| output<< (reduce i,m) }
+  node_types.each{|nt| output<< (nonterminal nt) }
+  map_with_index(all_states){|st,i| output<< (state st,i) }
+  #output<< error_handler  #disabled, i have rules for error recovery
+  output<< "}"
+end
+
+
+
+
+=begin
+4.5 Error Recovery
+yacc's error recovery mechanism is rather idiosyncratic. In fact, examining two books, [LMB92]
+and [ASU86], and the output generated by yacc yields three dierent descriptions of the recovery
+mechanism. We have tried to be faithful to the output of yacc.
+Fortunately, the mechanism has few consequences to the generation of the rest of the hard-
+coded parser. The only change to the parser is the maintenance of the variable, yyerrorstatus.
+Although relatively short, the code below is very subtle, like the explanation of yacc's error
+recovery mechanism. The code is given only for completeness.
+=end
+
+def error_handler
+%[
+error_handler:
+  if (yyerrorstatus > 2){
+    yyerror("syntax error");
+  }
+user_error_handler:
+  if (yyerrorstatus == 0){
+    huh if (la_identity == 0) YYABORT;// End of input.
+    la_identity = yylex(&la_token);
+    switch (OLDSTACK){
+    #{@states.map{|state| 
+        i=state.small_int
+        "case #{i}: goto state_action_#{str2cname state.name};\n"
+      }
+    }
+  }else{
+    yyerrorstatus = 0;
+    while (stack != stack_start){
+      switch (OLDSTACK){
+      case N: goto state_M;// iff M = goto[N,error].
+      .
+      .
+      .
+      }
+      stack--;
+    }
+    YYABORT;// Empty stack.
+  }
+]
+end
+
+
+  CHARMAPPINGS={
+  ?`=>'bquote',  ?~=>'tilde',  ?!=>'bang',    ?@=>'at',
+  ?#=>'num',     ?$=>'dollar', ?%=>'percent', ?^=>'caret',
+  ?&=>'and',     ?*=>'star',   ?(=>'lparen',  ?)=>'rparen',
+  ?-=>'minus',   ?+=>'plus',   ?==>'equals',
+  ?{=>'lbrace',  ?}=>'rbrace', ?[=>'lbrack',  ?]=>'rbrack',
+  ?|=>'or',      ?\\=>'bslash',?:=>'colon',   ?;=>'semicolon',
+  ?"=>'dquote',  ?'=>'squote', ?,=>'comma',   ?.=>'dot',
+  ?<=>'less',    ?>=>'more',   ??=>'q',       ?/=>'y',
+  ?\s=>'space',
+  ?X=>'x',
+  }
+  STRMAPPINGS={
+    '::'=>"XX",
+    '++'=>"Xeval",
+    '--'=>"Xsingleton",
+    '[]'=>"Xbrackets",
+    '->'=>"Xcalling",
+  }
+  STRMAP_REX=/#{STRMAPPINGS.keys.map{|x| Regexp.quote x}.join "|"}/
+  def self.str2cname str
+    str.gsub(STRMAP_REX){|str| STRMAPPINGS[str] } \
+       .gsub(/(?!#{LETTER_DIGIT}).|[X]/o){|ch| 
+         "X"+  esc=CHARMAPPINGS[ch[0]] ? esc : ch[0].to_s(16)
+       } 
+  end
+  def str2cname(str) RedParse.str2cname(str) end
+
+
+=begin
+The case arms are the only part of the code that depends on the automaton. Any state that has
+an outgoing transition on yacc's special error symbol will have a case arm in the second switch
+statement.
+
+Our error recovery implementation assumes that at most
+MAX UINT-3
+tokens will ever be shifted between syntactic
+errors | given that this number is 4,294,967,293 on a 32-bit machine, we feel this is safe. Even then, the mechanism
+is only flawed for the last 3 tokens out of every
+MAX UINT
+tokens, and furthermore, the assumption cannot disturb
+a correct parse, only error-recovery processing. Making it completely safe would be trivial, but would require a
+conditional increment at each shift, which we consider too costly for the benefit.
+=end
+end

data/lib/redparse/node.rb

@@ -17,7 +17,12 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 =end
 
-require 'rubygems'
+begin
+  require 'rubygems'
+rescue LoadError=>e
+  raise unless /rubygems/===e.message
+  #hope we don't need it
+end
 require 'tempfile'
 require 'pp'
 require "rubylexer"
@@ -35,20 +40,13 @@ class RedParse
       self::const_set k,t if Module===t and RubyLexer::Token>=t
     }
 
-    module SimpleToLisp
-      def to_lisp; to_s end
-    end
-
-    [NumberToken, SymbolToken, VarNameToken, MethNameToken].each{|tokclass|
-      tokclass.send :include, SimpleToLisp
-    }
-
     module FlattenedIvars
       def flattened_ivars
         result=[]
         instance_variables.sort.each{|iv| 
-          break if iv=="@data"
-          result.push iv, instance_variable_get(iv)
+          if iv!="@data"
+            result.push iv, instance_variable_get(iv)
+          end
         }
         return result
       end
@@ -59,28 +57,139 @@ class RedParse
       end
     end
 
+    module Stackable
+      module Meta
+        #declare name to be part of the identity of current class
+        #variations are the allowed values for name in this class
+        #keep variations simple: booleans, integers, symbols and strings only
+        def identity_param name, *variations
+          name=name.to_s
+          list=
+          if (variations-[true,false,nil]).empty?
+            #const_get("BOOLEAN_IDENTITY_PARAMS") rescue const_set("BOOLEAN_IDENTITY_PARAMS",{})
+              self.boolean_identity_params
+          else
+            #const_get("IDENTITY_PARAMS") rescue const_set("IDENTITY_PARAMS",{})
+              self.identity_params
+          end
+          list[name]=variations
+          return #old way to generate examplars below
+=begin
+          old_exemplars=self.exemplars||=[allocate]
+          exemplars=[]
+          variations.each{|var| old_exemplars.each{|exm| 
+            exemplars<< res=exm.dup
+
+            #res.send name+"=", var
+            #res.send :define_method, name do var end
+            Symbol|String|Integer|true|false|nil===var or fail #so inspect works as a quoting 
+            eval "def res.#{name}; #{var.inspect} end"
+          }}
+          old_exemplars.replace exemplars
+=end
+        end
+        def enumerate_exemplars
+          @exemplars||= build_exemplars
+        end
+        def build_exemplars
+          exemplars=[[self]]
+
+          (boolean_identity_params.merge identity_params).each{|name,variations|
+            todo=[]
+            variations=variations.dup
+            variations.each{|var|
+              exemplars.each{|exm|
+                res=exm.dup
+
+                #res.send name+"=", var
+                #res.send :define_method, name do var end
+                Symbol|String|Integer|true|false|nil===var or fail #so inspect works as a quoting
+                #eval "def res.#{name}; #{var.inspect} end"
+                res.push name, var
+
+                todo<<res #unless exemplars.include? res
+              }
+            }
+            exemplars=todo
+          }
+          #by now, may be some exemplars with identical identities...
+          #those duplicate identities should be culled
+#          identities_seen={}
+#          exemplars.delete_if{|ex|
+#            idn=ex.identity_name
+#            chuck_it=identities_seen[idn]
+#            identities_seen[idn]=true
+#            chuck_it
+#          }
+
+          return exemplars
+        end
+        attr_writer :boolean_identity_params, :identity_params
+        def identity_params
+          return @identity_params if defined?(@identity_params) and @identity_params
+          @identity_params=
+          if superclass.respond_to? :identity_params
+            superclass.identity_params.dup
+          else
+            {}
+          end
+        end
+        def boolean_identity_params
+          return @boolean_identity_params if defined?(@boolean_identity_params) and @boolean_identity_params
+          @boolean_identity_params=
+          if superclass.respond_to? :boolean_identity_params
+            superclass.boolean_identity_params.dup
+          else
+            {}
+          end
+        end
+      end #of Meta
+
+      def identity_name
+        k=self.class
+        list=[k.name]
+        list.concat k.boolean_identity_params.map{|(bip,)| bip if send(bip) }.compact
+        list.concat k.identity_params.map{|(ip,variations)|
+          val=send(ip)
+          variations.include? val or fail "identity_param #{k}##{ip} unexpected value #{val.inspect}"
+          [ip,val] 
+        }.flatten
+        result=list.join("_")
+        return result
+      end
+    end
+
     class Token
+      include Stackable
+      extend Stackable::Meta
+
       def image; "#{inspect}" end
  
-      def to_parsetree(*options)
-        options.include? :newlines and Thread.current[:$RedParse_parsetree_newlines]=true
-        options.include? :quirks and Thread.current[:$RedParse_parsetree_quirks]=true        
-
-        result=[parsetree] 
+      def to_parsetree(*options) #this shouldn't be needed anymore
+        o={}
+        [:newlines,:quirks,:ruby187].each{|opt| 
+          o[opt]=true if options.include? opt
+        }
 
-        Thread.current[:$RedParse_parsetree_newlines]=false
-        Thread.current[:$RedParse_parsetree_quirks]=false
+        result=[parsetree(o)] 
 
         result=[] if result==[[]]
 
         return result
       end
-      def lvalue?; nil end
+      def lvalue; nil end
       def data; [self] end
       def unary; false end
-      def rescue_parsetree; parsetree end
-      def begin_parsetree; parsetree end
+      def rescue_parsetree(o); parsetree(o) end
+      def begin_parsetree(o); parsetree(o) end
 
+      attr :line
+      alias endline line
+
+      attr_writer :startline
+      def startline
+        @startline||=endline
+      end
     end
 
     class KeywordToken
@@ -92,30 +201,91 @@ class RedParse
         @not_real if defined? @not_real
       end
 
+      identity_param :ident, *%w<+@ -@ unary& unary* ! ~ not defined?>+ #should be unary ops
+                               %w<end ( ) { } [ ] alias undef in>+
+                               %w<? : ; !~ lhs, rhs, rescue3>+ #these should be ops
+                               %w{*= **= <<= >>= &&= ||= |= &= ^= /= %= -= += = => ... .. . ::}+ #shouldn't be here, grrr
+                               RubyLexer::FUNCLIKE_KEYWORDLIST+
+                               RubyLexer::VARLIKE_KEYWORDLIST+
+                               RubyLexer::INNERBOUNDINGWORDLIST+
+                               RubyLexer::BINOPWORDLIST+
+                               RubyLexer::BEGINWORDLIST
+      #identity_param :unary, true,false,nil
+      
+      #identity_param :tag, :lhs,:rhs,:param,:call,:array,:block,:nested,nil
+      identity_param :callsite?, nil, true, false
+      identity_param :not_real?, nil, true, false
+      identity_param :infix, nil, true
       alias image ident
+
+
+      #KeywordToken#as/infix should be in rubylexer
+      alias old_as as
+      def as
+        if tag and ident[/^[,*&]$/]
+          tag.to_s+ident
+        else old_as
+        end
+      end
+
+      def infix
+        @infix if defined? @infix
+      end unless instance_methods.include? "infix"
     end
 
-    class VarNameToken
+    class OperatorToken
+      identity_param :ident, *%w[+@ -@ unary& unary* lhs* ! ~ not defined? * ** + - 
+                                 < << <= <=> > >= >> =~ == ===
+                                 % / & | ^ != !~ = => :: ? : , ; . .. ... 
+                                 *= **= <<= >>= &&= ||= && ||
+                                 &= |= ^= %= /= -= += and or
+                              ]+RubyLexer::OPORBEGINWORDLIST+%w<; lhs, rhs, rescue3>
+      #identity_param :unary, true,false,nil
+      #identity_param :tag, :lhs,:rhs,:param,:call,:array,:block,:nested,nil
+
+      #this should be in rubylexer
+      def as
+        if tag and ident[/^[,*&]$/]
+          tag.to_s+ident
+        end
+      end
+    end
+
+    class NumberToken
+      alias to_lisp to_s
+      def negative; /\A-/ === ident end unless instance_methods.include? "negative"
+
+      identity_param :negative, true,false
+    end
+
+    class MethNameToken
+      alias to_lisp to_s
+      def has_equals; /#{LETTER_DIGIT}=$/o===ident end unless instance_methods.include? "has_equals"
+
+      identity_param :has_equals, true,false
+    end
+
+    class VarNameToken #none of this should be necessary now
       include FlattenedIvars
       alias image ident
 
       alias == flattened_ivars_equal?
 
-      def parsetree 
+      def parsetree(o)
         type=case ident[0]
-         when ?$: 
+         when ?$ 
            case ident[1]
-           when ?1..?9: return [:nth_ref,ident[1..-1].to_i]
-           when ?&,?+,?`,?': return [:back_ref,ident[1].chr.to_sym] #`
+           when ?1..?9; return [:nth_ref,ident[1..-1].to_i]
+           when ?&,?+,?`,?'; return [:back_ref,ident[1].chr.to_sym] #`
            else :gvar
            end
-         when ?@: 
+         when ?@ 
            if ident[1]==?@
              :cvar
            else
              :ivar
            end
-         when ?A..?Z: :const
+         when ?A..?Z; :const
          else 
            case lvar_type
            when :local; :lvar
@@ -129,28 +299,28 @@ class RedParse
 
       def varname2assigntype
           case ident[0]
-            when ?$: :gasgn
-            when ?@: 
-              if ident[1]!=?@:  :iasgn
-              elsif in_def:     :cvasgn
-              else              :cvdecl
-              end
-            when ?A..?Z: :cdecl
-            else :lasgn
-              case lvar_type
-              when :local; :lasgn
-              when :block; :dasgn
-              when :current; :dasgn_curr
-              else fail
-              end
+          when ?$; :gasgn
+          when ?@;
+            if ident[1]!=?@;  :iasgn
+            elsif in_def;     :cvasgn
+            else              :cvdecl
+            end
+          when ?A..?Z; :cdecl
+          else 
+            case lvar_type
+            when :local; :lasgn
+            when :block; :dasgn
+            when :current; :dasgn_curr
+            else fail
+            end
           end
       end
 
-      def lvalue_parsetree
+      def lvalue_parsetree(o)
         [varname2assigntype, ident.to_sym]
       end
 
-      def lvalue?
+      def lvalue
         return @lvalue if defined? @lvalue
         @lvalue=true
       end
@@ -159,7 +329,7 @@ class RedParse
         lvar_type==:current ? [ident] : []
       end
 
-      attr_accessor :line,:lvalue
+      attr_accessor :endline,:lvalue
 
       def dup
         result=super
@@ -169,7 +339,7 @@ class RedParse
 
       public :remove_instance_variable
 
-      def unparse o; ident end
+      def unparse o=default_unparse_options; ident end
       alias lhs_unparse unparse
 
       def delete_extraneous_ivars!
@@ -177,21 +347,24 @@ class RedParse
       end
 
       def walk
-         yield nil,nil,nil,self
+        yield nil,nil,nil,self
       end
     end   
 
     class StringToken
       attr :char
     end
-  
+
     class HerePlaceholderToken
       attr_accessor :node
       attr :string
     end
 
     class Node<Array
+      include Stackable
+      extend Stackable::Meta
       include FlattenedIvars
+
       def initialize(*data)
         replace data
       end
@@ -206,13 +379,14 @@ class RedParse
         result
       end
 
-
       def ==(other)
         super and flattened_ivars_equal?(other)
       end
 
       def image; "(#{inspect})" end
 
+      def error? x; false end
+
       @@data_warned=nil
       def data
         unless @@data_warned
@@ -223,7 +397,11 @@ class RedParse
       end
       alias unwrap data
 
-      attr_accessor :line
+      attr_writer :startline
+      def startline
+        @startline||=endline
+      end
+      attr_accessor :endline
       attr_accessor :errors
 
       def self.[](*data)
@@ -304,27 +482,32 @@ class RedParse
       def lhs_unparse o; unparse(o) end
 
       def to_parsetree(*options)
-        options.include? :newlines and Thread.current[:$RedParse_parsetree_newlines]=true
-        options.include? :quirks and Thread.current[:$RedParse_parsetree_quirks]=true        
-
-        result=[parsetree] 
+        o={}
+        [:newlines,:quirks,:ruby187].each{|opt| 
+          o[opt]=true if options.include? opt
+        }
 
-        Thread.current[:$RedParse_parsetree_newlines]=false
-        Thread.current[:$RedParse_parsetree_quirks]=false
+        result=[parsetree(o)] 
 
-        result=[] if result==[[]]
+        result=[] if result==[[]] || result==[nil]
 
         return result
       end
-      def parsetree
+
+      def to_parsetree_and_warnings(*options)
+        #for now, no warnings are ever output
+        return to_parsetree(*options),[]
+      end
+
+      def parsetree(o)
         "wrong(#{inspect})"
       end
 
-      def rescue_parsetree; parsetree end
-      def begin_parsetree; parsetree end
+      def rescue_parsetree(o); parsetree(o) end
+      def begin_parsetree(o); parsetree(o) end
 
-      def parsetrees list
-        !list.empty? and list.map{|node| node.parsetree}
+      def parsetrees list,o
+        !list.empty? and list.map{|node| node.parsetree(o)}
       end
 
       def negate(condition,offset=nil)
@@ -342,8 +525,8 @@ class RedParse
         callback[ parent,index,subindex,self ] and
         each_with_index{|datum,i|
           case datum
-          when Node: datum.walk(self,i,&callback)
-          when Array:
+          when Node; datum.walk(self,i,&callback)
+          when Array;
             datum.each_with_index{|x,j| 
               Node===x ? x.walk(self,i,j,&callback) : callback[self,i,j,x]
             }
@@ -352,11 +535,65 @@ class RedParse
         }
       end
 
+      def depthwalk(parent=nil,index=nil,subindex=nil,&callback)
+        each_with_index{|datum,i|
+          case datum
+          when Node
+            datum.depthwalk(self,i,&callback)
+          when Array
+            datum.each_with_index{|x,j| 
+              if Node===x
+                x.depthwalk(self,i,j,&callback) 
+              else 
+                callback[self,i,j,x]
+              end
+            }
+          else 
+            callback[self, i, nil, datum]
+          end
+        }
+        callback[ parent,index,subindex,self ]
+      end
+
+      def add_parent_links!
+        walk{|parent,i,subi,o|
+          o.parent=parent if Node===o
+        }
+      end
+
+      attr_accessor :parent
+
+      def xform_tree!(xformer)
+        session={}
+        depthwalk{|parent,i,subi,o|
+          xformer.xform!(o,session) if o
+        }
+        session["final"]=true
+        depthwalk{|parent,i,subi,o|
+          if session.has_key? o.__id__
+            new= session[o.__id__]
+            if Reg::Formula===new
+              new=new.formula_value(o,session)
+            end
+            subi ? parent[i][subi]=new : parent[i]=new
+          end
+        }
+        if session.has_key? self.__id__
+          new= session[self.__id__]
+          if Reg::Formula===new
+            new=new.formula_value(self,session)
+          end
+          return new
+        else
+          return self
+        end
+      end
+
       def linerange
         min=9999999999999999999999999999999999999999999999999999
         max=0
         walk{|parent,i,subi,node|
-          if node.respond_to? :line and line=node.line
+          if node.respond_to? :endline and line=node.endline
             min=[min,line].min
             max=[max,line].max
           end
@@ -364,7 +601,7 @@ class RedParse
         return min..max
       end
 
-      def fixup_multiple_assignments!
+      def fixup_multiple_assignments! #dead code
        result=self
        walk{|parent,i,subi,node|
         if CommaOpNode===node
@@ -413,18 +650,18 @@ class RedParse
 
       def prohibit_fixup x
         case x
-        when UnaryStarNode: true
-#        when ParenedNode: x.size>1
-        when CallSiteNode: x.params and !x.real_parens
+        when UnaryStarNode; true
+#        when ParenedNode; x.size>1
+        when CallSiteNode; x.params and !x.real_parens
         else false
         end
       end
 
-      def fixup_rescue_assignments!
+      def fixup_rescue_assignments! #dead code
         result=self
         walk{|parent,i,subi,node|
           #if a rescue op with a single assignment on the lhs
-          if ParenedNode===node and node.op? and assign=node.first and #ick
+          if RescueOpNode===node and assign=node.first and #ick
                AssignNode===assign and assign.op.ident=="=" and 
                !(assign.multi? or 
                  prohibit_fixup assign.right)
@@ -432,7 +669,7 @@ class RedParse
 
             #re-hang the node with = at the top instead of rescue
             node=AssignNode.new(assign.left, assign.op,
-              ParenedNode.new(assign.right,nil,node[1][0].action)
+              RescueOpNode.new(assign.right,nil,node[1][0].action)
             )
             
             #graft the new node back onto the old tree
@@ -461,7 +698,7 @@ class RedParse
         result=[]
         walk {|parent,i,subi,node|
           case node
-          when MethodNode,ClassNode,ModuleNode,MetaClassNode: false
+          when MethodNode,ClassNode,ModuleNode,MetaClassNode; false
           when CallSiteNode
             Node===node.receiver and
               result.concat node.receiver.lvars_defined_in 
@@ -474,19 +711,17 @@ class RedParse
             lvalue.respond_to? :all_current_lvars and
               result.concat lvalue.all_current_lvars 
             true
-          when ForNode: 
+          when ForNode
             lvalue=node.for
             lvalue.respond_to? :all_current_lvars and
               result.concat lvalue.all_current_lvars 
             true
-          when ParenedNode: 
-            if node.size>1
+          when RescueOpNode,BeginNode
               rescues=node[1]
               rescues.each{|resc|
                 name=resc.varname
                 name and result.push name.ident
               }
-            end
             true
           else true
           end
@@ -497,7 +732,7 @@ class RedParse
       end
 
       def unary; false end
-      def lvalue?; false end
+      def lvalue; nil end
 
       def deep_copy transform={},&override
         handler=proc{|child|
@@ -537,10 +772,11 @@ class RedParse
       def delete_extraneous_ivars!
         walk{|parent,i,subi,node|
           case node
-          when Node,VarNameToken
+          when Node
             node.remove_instance_variable :@offset rescue nil
             node.remove_instance_variable :@loopword_offset rescue nil
-            node.remove_instance_variable :@line rescue nil
+            node.remove_instance_variable :@endline rescue nil
+            node.remove_instance_variable :@lvalue rescue nil
             if node.respond_to? :lvalue 
               node.lvalue or
                 node.remove_instance_variable :@lvalue rescue nil 
@@ -554,6 +790,18 @@ class RedParse
         return self
       end
 
+      def delete_linenums!
+        walk{|parent,i,subi,node|
+          case node
+          when Node
+            node.remove_instance_variable :@endline rescue nil
+            node.remove_instance_variable :@startline rescue nil
+          end
+          true
+        }
+        return self
+      end
+
       public :remove_instance_variable
 
       #convert to a Reg::Array expression. subnodes are also converted.
@@ -575,9 +823,6 @@ class RedParse
       end
 
     private
-      def quirks
-        Thread.current[:$RedParse_parsetree_quirks]
-      end
 
       #turn a list (array) of arrays into a linked list, in which each array
       #has a reference to the next in turn as its last element.
@@ -620,25 +865,151 @@ class RedParse
         handle_normals[]
       end
 
-      def param_list_parse(param_list)
+      def param_list_parse(param_list,o)
         output=[]
         star=amp=nil
         param_list_walk(param_list){|type,val,i|
           case type
-          when '':
-            output.concat val.map{|param| param.rescue_parsetree}
-          when '=>':
-            output.push HashLiteralNode.new(nil,val,nil).parsetree
-          when '*': star=val.parsetree
-          when '&': amp=val.parsetree
+          when ''
+            output.concat val.map{|param| param.rescue_parsetree(o)}
+          when '=>'
+            output.push HashLiteralNode.new(nil,val,nil).parsetree(o)
+          when '*'; star=val.parsetree(o)
+          when '&'; amp=val.parsetree(o)
           end
         }
         return output,star,amp
       end
+
+      def unparse_nl(token,o,alt=';',nl="\n")
+
+        #should really only emit newlines 
+        #to bring line count up to startline, not endline.
+
+        linenum= Integer===token ? token : token.startline rescue (return alt)
+        shy=(linenum||0)-o[:linenum]
+        return alt if shy<=0
+        o[:linenum]=linenum
+        return nl*shy
+      end
+
+      def default_unparse_options
+        {:linenum=>1}
+      end
     end
 
     class ValueNode<Node
-      def lvalue?; nil end
+      def lvalue; nil end
+      #identity_param :lvalue, nil, true
+    end
+
+    class VarNode<ValueNode
+      include FlattenedIvars
+      attr_accessor :endline
+      attr_reader :lvar_type,:in_def,:offset
+      attr_writer :lvalue
+
+
+      alias == flattened_ivars_equal?
+
+      def initialize(tok)
+        super(tok.ident)
+        @lvar_type=tok.lvar_type
+        @offset=tok.offset
+        @endline=tok.endline
+        @in_def=tok.in_def
+      end
+
+      def ident; first end
+      def ident=x; self[0]=x end
+      alias image ident
+      alias startline endline
+
+      def parsetree(o)
+        type=case ident[0]
+         when ?$: 
+           case ident[1]
+           when ?1..?9; return [:nth_ref,ident[1..-1].to_i]
+           when ?&,?+,?`,?'; return [:back_ref,ident[1].chr.to_sym] #`
+           else :gvar
+           end
+         when ?@
+           if ident[1]==?@
+             :cvar
+           else
+             :ivar
+           end
+         when ?A..?Z; :const
+         else 
+           case lvar_type
+           when :local; :lvar
+           when :block; :dvar
+           when :current; :dvar#_curr
+           else fail
+           end
+         end
+         return [type,ident.to_sym]
+      end
+
+      def varname2assigntype
+          case ident[0]
+            when ?$; :gasgn
+            when ?@
+              if ident[1]!=?@;  :iasgn
+              elsif in_def;     :cvasgn
+              else              :cvdecl
+              end
+            when ?A..?Z; :cdecl
+            else
+              case lvar_type
+              when :local; :lasgn
+              when :block; :dasgn
+              when :current; :dasgn_curr
+              else fail
+              end
+          end
+      end
+
+      def lvalue_parsetree(o)
+        [varname2assigntype, ident.to_sym]
+      end
+
+      alias to_lisp to_s
+
+      def lvalue
+        return @lvalue if defined? @lvalue
+        @lvalue=true
+      end
+
+      identity_param :lvalue, nil, true
+
+      def all_current_lvars
+        lvar_type==:current ? [ident] : []
+      end
+
+
+      def dup
+        result=super
+        result.ident=@ident.dup if @ident
+        return result
+      end
+
+      public :remove_instance_variable
+
+      def unparse o=default_unparse_options; ident end
+      alias lhs_unparse unparse
+
+#      def delete_extraneous_ivars!
+#        remove_instance_variable :@lvalue
+#        super
+#      end
+
+if false
+      def walk #is this needed?
+         yield nil,nil,nil,self
+      end
+end
+
     end
 
     #forward decls
@@ -704,34 +1075,34 @@ class RedParse
         "(#{op} #{left.to_lisp} #{right.to_lisp})"
       end
 
-      def parsetree
+      def parsetree(o)
         [:call, 
-           left.rescue_parsetree, 
+           left.rescue_parsetree(o), 
            op.to_sym, 
-           [:array, right.rescue_parsetree]
+           [:array, right.rescue_parsetree(o)]
         ]
       end
       alias opnode_parsetree parsetree
 
-      def unparse o
+      def unparse o=default_unparse_options
         result=l=left.unparse(o)
-        result+=" " if /a-z_/i===op
+        result+=" " if /\A(?:!|#{LCLETTER})/o===op
         result+=op
-        result+=" " if /a-z_/i===op or / \Z/===l
+        result+=" " if /#{LETTER_DIGIT}\Z/o===op or / \Z/===l
         result+=right.unparse(o)      
       end
 
-      def unparse o; raw_unparse o end
+#      def unparse o=default_unparse_options; raw_unparse o end
     end
 
     module MatchNode
       include OpNode
 
-      def parsetree
+      def parsetree(o)
         if StringNode===left and left.char=='/'
-          [:match2, left.parsetree, right.parsetree]
+          [:match2, left.parsetree(o), right.parsetree(o)]
         elsif StringNode===right and right.char=='/'
-          [:match3, right.parsetree, left.parsetree]
+          [:match3, right.parsetree(o), left.parsetree(o)]
         else
           super 
         end
@@ -742,8 +1113,8 @@ class RedParse
     module NotEqualNode
       include OpNode
 
-      def parsetree
-        result=opnode_parsetree
+      def parsetree(o)
+        result=opnode_parsetree(o)
         result[2]="=#{op[1..1]}".to_sym
         result=[:not, result]
         return result
@@ -754,11 +1125,11 @@ class RedParse
     module NotMatchNode
       include NotEqualNode
 
-      def parsetree
+      def parsetree(o)
         if StringNode===left and left.char=="/"
-          [:not, [:match2, left.parsetree, right.parsetree]]
+          [:not, [:match2, left.parsetree(o), right.parsetree(o)]]
         elsif StringNode===right and right.char=="/"
-          [:not, [:match3, right.parsetree, left.parsetree]]
+          [:not, [:match3, right.parsetree(o), left.parsetree(o)]]
         else
           super
         end
@@ -775,11 +1146,11 @@ class RedParse
                [val1]
              end
         if self.class===val2
-          list.push *val2
+          list.push( *val2 )
         elsif val2
           list.push val2
         end 
-        super *list
+        super( *list )
       end
     end
 
@@ -788,11 +1159,12 @@ class RedParse
       def to_lisp
         "(#{map{|x| x.to_lisp}.join(" ")})"
       end
-      def lvalue?
+      def lvalue
         return @lvalue if defined? @lvalue
         @lvalue=true
       end
-      attr_accessor :lvalue
+      attr_writer :lvalue
+      identity_param :lvalue, nil, true
     end
 
     class LiteralNode<ValueNode; end
@@ -818,24 +1190,26 @@ class RedParse
       #class<<LITCAT; alias === call; end
       LITCAT=StringCatNode& item_that.grep(~LITSTR).empty? #+[LITSTR.+]
       LITNODE=LiteralNode|NopNode|LITSTR|LITCAT|LITRANGE|(VarLikeNode&-{:name=>/^__/})
-            #VarNameToken|  #why not this too?
-      def parsetree
+            #VarNode|  #why not this too?
+      def parsetree(o)
         data=compact
         data.empty? and return
         items=Array.new(data[0...-1])
-        if quirks
+        if o[:quirks]
           items.shift while LITNODE===items.first
         else
           items.reject!{|expr| LITNODE===expr }
         end
-        items.map!{|expr| expr.rescue_parsetree}.push last.parsetree
-#        items=map{|expr| expr.parsetree}
+        items.map!{|expr| expr.rescue_parsetree(o)}.push last.parsetree(o)
+#        items=map{|expr| expr.parsetree(o)}
         items.reject!{|expr| []==expr }
-        if quirks
-          header=items.first
-          (items[0,1] = *header[1..-1]) if header and header.first==:block
+        if o[:quirks] 
+          unless BeginNode===data[0]
+            header=items.first
+            (items[0,1] = *header[1..-1]) if header and header.first==:block
+          end
         else
-          items.size.pred.downto(0){|i|
+          (items.size-1).downto(0){|i|
             header=items[i]
             (items[i,1] = *header[1..-1]) if header and header.first==:block
           }     
@@ -849,25 +1223,21 @@ class RedParse
         end
       end
 
-      def unparse o
-        map{|expr| expr.unparse(o)}.join("\n")      
+      def unparse o=default_unparse_options
+        map{|expr| unparse_nl(expr,o)+expr.unparse(o)}.to_s
       end
     end
 
     class StringCatNode < ValueNode
-      def initialize(strs,str1,str2)
-        if HereDocNode===strs
-          hd,strs,str=strs,str1,str2
-          strs.push str
-        else
-          strs.push str1,str2
-        end
+      def initialize(*strses)
+        strs=strses.pop.unshift( *strses )
+        hd=strs.shift if HereDocNode===strs.first
         strs.map!{|str| StringNode.new(str)}
         strs.unshift hd if hd
-        super *strs
+        super( *strs )
       end
-      def parsetree
-        result=map{|str| str.parsetree}
+      def parsetree(o)
+        result=map{|str| str.parsetree(o)}
         sum=''
         type=:str
         tree=i=nil
@@ -885,6 +1255,8 @@ class RedParse
               x[0]=[:str,x0]
             end
             cat+x
+          elsif x.last.empty?
+            cat
           else
             cat+[x]
           end
@@ -892,7 +1264,7 @@ class RedParse
         ]
       end
 
-      def unparse o
+      def unparse o=default_unparse_options
         map{|ss| ss.unparse(o)}.join ' '      
       end
     end
@@ -900,10 +1272,14 @@ class RedParse
 #    class ArrowOpNode<ValueNode
 #      param_names(:left,:arrow_,:right)
 #    end
-     module ArrowOpNode #not to appear in final tree?
+     module ArrowOpNode
        def initialize(*args) 
          @module=ArrowOpNode
        end
+
+       def unparse(o=default_unparse_options)
+         left.unparse(o)+" => "+right.unparse(o)
+       end
      end
 
 #    class RangeNode<ValueNode
@@ -921,9 +1297,9 @@ class RedParse
       def last; right end
       def exclude_end?; @exclude_end end
 
-      def parsetree
-        first=first().parsetree
-        last=last().parsetree
+      def parsetree(o)
+        first=first().parsetree(o)
+        last=last().parsetree(o)
         if :lit==first.first and :lit==last.first and
            Fixnum===first.last and Fixnum===last.last
           return [:lit, Range.new(first.last,last.last,@exclude_end)]
@@ -938,7 +1314,7 @@ class RedParse
         @as_flow_control=true
       end
 
-      def unparse(o)
+      def unparse(o=default_unparse_options)
         result=left.unparse(o)+'..'
         result+='.' if exclude_end?
         result << right.unparse(o)
@@ -949,9 +1325,9 @@ class RedParse
     class UnOpNode<ValueNode
       param_names(:op,:val)
       def initialize(op,val)
-#        /^[&*+-]$/===op.ident and op.ident+="@"
         op=op.ident
-        /^(!|not)$/===op and 
+        /([&*])$/===op and op=$1+"@"
+        /^(?:!|not)$/===op and 
           val.respond_to? :special_conditions! and 
             val.special_conditions!
         super(op,val)
@@ -961,49 +1337,54 @@ class RedParse
 
       def image; "(#{op})" end
 
-      def lvalue?
+      def lvalue
 #        return nil unless op=="*@"
 
         return @lvalue if defined? @lvalue
         @lvalue=true
       end
-      attr_accessor :lvalue
+      attr_writer :lvalue
 
       def to_lisp
         "(#{op} #{val.to_lisp})"
       end
 
-      def parsetree
+      def parsetree(o)
         node=self
         node=node.val while UnOpNode===node and node.op=="+@"
-        return node.parsetree if LiteralNode&-{:val=>Integer|Float|Symbol}===node
-        return node.parsetree if StringNode&-{:char=>'/', :size=>1}===node
+        return node.parsetree(o) if LiteralNode&-{:val=>Integer|Float|Symbol}===node
+        return node.parsetree(o) if StringNode&-{:char=>'/', :size=>1}===node
 
         case op
-        when /^&/: [:block_arg, val.ident.to_sym]
-        when "!","not": [:not, val.rescue_parsetree]
-        when "defined?": [:defined, val.parsetree]
+        when /^&/; [:block_arg, val.ident.to_sym]
+        when "!","not"; [:not, val.rescue_parsetree(o)]
+        when "defined?"; [:defined, val.parsetree(o)]
         else
-          [:call, val.rescue_parsetree, op.to_sym]
+          [:call, val.rescue_parsetree(o), op.to_sym]
         end
       end
 
-      def lvalue_parsetree
-        parsetree
+      def lvalue_parsetree(o)
+        parsetree(o)
       end
 
-      def unparse o
+      def unparse o=default_unparse_options
         op=op()
         op=op.chomp "@"
         result=op
-        result+=" " if /[a-z_]/i===op
+        result+=" " if /#{LETTER}/o===op or /^[+-]/===op && LiteralNode===val
         result+=val.unparse(o)
       end
     end
 
     class UnaryStarNode<UnOpNode
-      def parsetree
-        [:splat, val.rescue_parsetree]
+      def initialize(op,val)
+        op.ident="*@"
+        super(op,val)
+      end
+
+      def parsetree(o)
+        [:splat, val.rescue_parsetree(o)]
       end
 
       def all_current_lvars
@@ -1012,11 +1393,13 @@ class RedParse
       end
       attr_accessor :after_comma
 
-      def lvalue_parsetree
-        val.lvalue_parsetree
+      def lvalue_parsetree o
+        val.lvalue_parsetree(o)
       end
 
-      def unparse o
+      identity_param :lvalue, nil, true
+
+      def unparse o=default_unparse_options
         "*"+val.unparse(o)      
       end
     end
@@ -1025,14 +1408,14 @@ class RedParse
       #param_names :op,:val
       def initialize(star)
         @offset= star.offset
-        replace ['*@',VarNameToken.new('',offset)]
+        replace ['*@',VarNode.new(VarNameToken.new('',offset))]
       end
       attr :offset
       def lvars_defined_in; [] end
-      def parsetree; [:splat] end
+      def parsetree(o); [:splat] end
       alias lvalue_parsetree parsetree
 
-      def unparse(o); "* "; end
+      def unparse(o=nil); "* "; end
     end
 
     class DanglingCommaNode<DanglingStarNode
@@ -1041,26 +1424,26 @@ class RedParse
       end
       attr_accessor :offset
 
-      def lvalue_parsetree
+      def lvalue_parsetree o
         :dangle_comma
       end
       alias parsetree lvalue_parsetree
 
-      def unparse o;  "";   end
+      def unparse o=default_unparse_options;  "";   end
     end
 
     class ConstantNode<ListOpNode
       def initialize(*args)
         args.unshift nil if args.size==2
         args.map!{|node| 
-          if VarNameToken===node and (?A..?Z)===node.ident[0]
+          if VarNode===node and (?A..?Z)===node.ident[0]
           then node.ident 
           else node
           end
         }
         super(*args)  
       end
-      def unparse(o)
+      def unparse(o=default_unparse_options)
         if Node===first
           result=dup
           result[0]= first.unparse(o)#.gsub(/\s+\Z/,'')
@@ -1069,16 +1452,16 @@ class RedParse
         end
       end
       alias image unparse
-      def lvalue_parsetree
-        [:cdecl,parsetree]
+      def lvalue_parsetree(o)
+        [:cdecl,parsetree(o)]
       end
-      def parsetree
+      def parsetree(o)
         if !first
           result=[:colon3, self[1].to_sym]
           i=2
         else
           result=first.respond_to?(:parsetree) ? 
-                   first.parsetree :
+                   first.parsetree(o) :
                    [:const,first.to_sym]
           i=1
         end
@@ -1087,46 +1470,47 @@ class RedParse
         }
       end
 
-      def lvalue?
+      def lvalue
         return @lvalue if defined? @lvalue
         @lvalue=true
       end
-      attr_accessor :lvalue
+      attr_writer :lvalue
+      identity_param :lvalue, nil, true
     end
     LookupNode=ConstantNode
 
-    class DoubleColonNode<ValueNode #dunno about this name...     maybe ConstantNode?
-      #not used anymore
+    class DoubleColonNode<ValueNode       #obsolete
+         #dunno about this name...     maybe ConstantNode?
       param_names :namespace, :constant
       alias left namespace
       alias right constant
       def initialize(val1,op,val2=nil)
         val1,op,val2=nil,val1,op unless val2
-        val1=val1.ident if VarNameToken===val1 and /\A[A-Z]/===val1.ident
-        val2=val2.ident if VarNameToken===val2 and /\A[A-Z]/===val2.ident
+        val1=val1.ident if VarNode===val1 and /\A#{UCLETTER}/o===val1.ident
+        val2=val2.ident if VarNode===val1 and /\A#{UCLETTER}/o===val2.ident
         replace [val1,val2]
       end
 
       def image; '(::)' end
  
 
-      def parsetree
+      def parsetree(o)
         if namespace
-          ns= (String===namespace) ? [:const,namespace.to_sym] : namespace.parsetree
+          ns= (String===namespace) ? [:const,namespace.to_sym] : namespace.parsetree(o)
           [:colon2, ns, constant.to_sym]
         else
           [:colon3, constant.to_sym]
         end
       end
-      def lvalue_parsetree
-        [:cdecl,parsetree]
+      def lvalue_parsetree(o)
+        [:cdecl,parsetree(o)]
       end
 
-      def lvalue?
+      def lvalue
         return @lvalue if defined? @lvalue
         @lvalue=true
       end
-      attr_accessor :lvalue
+      attr_writer :lvalue
     end
 
     class DotCallNode<ValueNode #obsolete
@@ -1138,20 +1522,21 @@ class RedParse
         "(#{receiver.to_lisp} #{@data.last.to_lisp[1...-1]})"
       end
 
-      def parsetree
+      def parsetree(o)
         cs=self[1]
-        cs &&= cs.parsetree
+        cs &&= cs.parsetree(o)
         cs.shift if cs.first==:vcall or cs.first==:fcall
-        [:call, @data.first.parsetree, *cs]
+        [:call, @data.first.parsetree(o), *cs]
       end
-      def lvalue?
+      def lvalue
         return @lvalue if defined? @lvalue
         @lvalue=true
       end
-      attr_accessor :lvalue
+      attr_writer :lvalue
     end
 
-    class ParenedNode<ValueNode
+=begin
+    class OldParenedNode<ValueNode
       param_names :body, :rescues, :else!, :ensure!
       def initialize(*args)
         @empty_ensure=@op_rescue=nil
@@ -1197,20 +1582,10 @@ class RedParse
         huh #what about rescues, else, ensure?
         body.to_lisp
       end
-
-      def self.want_extra_begin!(x=true)
-        @@want_extra_begin=x
-      end
-      def want_extra_begin?
-        return @@want_extra_begin if defined? @@want_extra_begin
-        @@want_extra_begin =
-          defined?(::ParseTree) and 
-            ::ParseTree.new.parse_tree_for_string("begin; a; end")==[[:begin, [:vcall, :a]]]
-      end
  
-      def parsetree
+      def parsetree(o)
         if size==1
-          body.parsetree
+          body.parsetree(o)
         else
           body=body()
           target=result=[]   #was: [:begin, ]
@@ -1218,7 +1593,7 @@ class RedParse
           #body,rescues,else_,ensure_=*self
           target.push target=[:ensure, ] if ensure_ or @empty_ensure
 
-          rescues=rescues().map{|resc| resc.parsetree}
+          rescues=rescues().map{|resc| resc.parsetree(o)}
           if rescues.empty? 
             else_ and
               body=SequenceNode.new(body,nil,else_)
@@ -1228,66 +1603,309 @@ class RedParse
             else_=else_()
           end
           if body 
-              needbegin=  (ParenedNode===body  and want_extra_begin? and body.after_equals)
-              body=body.parsetree 
-              body=[:begin, body] if needbegin and body.first!=:begin
+              needbegin=  (BeginNode===body and body.after_equals)
+              body=body.parsetree(o) 
+              body=[:begin, body] if needbegin and body.first!=:begin and !o[:ruby187]
               (newtarget||target).push body if body
           end
-          target.push ensure_.parsetree if ensure_
+          target.push ensure_.parsetree(o) if ensure_
           target.push [:nil] if @empty_ensure
           target=newtarget if newtarget
           
           unless rescues.empty?
             target.push linked_list(rescues)
           end
-          target.push else_.parsetree if  else_ #and !body
+          target.push else_.parsetree(o) if  else_ #and !body
           result.size==0 and result=[[:nil]]
           result=result.last #if @op_rescue
+          result=[:begin,result] unless o[:ruby187]||op?||result==[:nil]#||result.first==:begin
           result
         end
       end
 
-#      def rescue_parsetree
-#        result=parsetree
-#        result.first==:begin and result=result.last
-#        result
-#      end
+      def rescue_parsetree o
+        result=parsetree o
+        result.first==:begin and result=result.last unless o[:ruby187]
+        result
+      end
 
-      def begin_parsetree
+      def begin_parsetree(o)
         body,rescues,else_,ensure_=*self
         needbegin=(rescues&&!rescues.empty?) || ensure_ || @empty_ensure
-        result=parsetree
-        needbegin and result=[:begin, result]
+        result=parsetree(o)
+        needbegin and result=[:begin, result] unless result.first==:begin
         result
       end
 
-      def lvalue?
+      def lvalue
         return nil unless size==1 
 #        case first
 #        when CommaOpNode,UnaryStarNode: #do nothing
-#        when ParenedNode: return first.lvalue?
+#        when ParenedNode: return first.lvalue
 #        else return nil
 #        end
 
         return @lvalue if defined? @lvalue
         @lvalue=true
       end
-      attr_accessor :lvalue
+      attr_writer :lvalue
 
-      def unparse(o)
+      def unparse(o=default_unparse_options)
         if size==1
           "("+(body&&body.unparse(o))+")"
         else
           result="begin "
           body&&result+= body.unparse(o)
-          result+="\n"
+          result+=unparse_nl(rescues.first,o)
           rescues.each{|resc| result+=resc.unparse(o) }
-          result+="\nensure "+ensure_.unparse(o) if ensure_
-          result+="\nelse "+else_.unparse(o) if else_
-          result+="\nend"
+          result+=unparse_nl(ensure_,o)+"ensure "+ensure_.unparse(o) if ensure_
+          result+=unparse_nl(else_,o)+"else "+else_.unparse(o) if else_
+          result+=";end"
         end
       end
     end
+=end
+
+    class ParenedNode<ValueNode
+      param_names :body #, :rescues, :else!, :ensure!
+      def initialize(lparen,body,rparen)
+        self[0]=body
+      end
+
+      attr_accessor :after_comma, :after_equals
+
+      def image; "(#{body.image})" end
+
+      def special_conditions!
+          node=body
+          node.special_conditions! if node.respond_to? :special_conditions!
+      end
+
+      def to_lisp
+        huh #what about rescues, else, ensure?
+        body.to_lisp
+      end
+
+      def op?; false end
+ 
+      def parsetree(o)
+          body.parsetree(o)
+      end
+
+      def rescue_parsetree o
+        body.rescue_parsetree o
+#        result.first==:begin and result=result.last unless o[:ruby187]
+#        result
+      end
+
+      alias begin_parsetree parsetree
+
+      def lvalue
+        return @lvalue if defined? @lvalue
+        @lvalue=true
+      end
+      attr_writer :lvalue
+
+      identity_param :lvalue, nil, true
+
+      def unparse(o=default_unparse_options)
+          "("+(body&&body.unparse(o))+")"
+      end
+    end
+
+    module HasRescue
+      def parsetree_and_rescues(o)
+          body=body()
+          target=result=[]   #was: [:begin, ]
+
+          #body,rescues,else_,ensure_=*self
+          target.push target=[:ensure, ] if ensure_ or @empty_ensure
+
+          rescues=rescues().map{|resc| resc.parsetree(o)}
+          if rescues.empty?
+            else_ and
+              body=SequenceNode.new(body,nil,else_)
+            else_=nil
+          else
+            target.push newtarget=[:rescue, ]
+            else_=else_()
+          end
+          if body
+#              needbegin=  (BeginNode===body and body.after_equals)
+              body=body.parsetree(o)
+#              body=[:begin, body] if needbegin and body.first!=:begin and !o[:ruby187]
+              (newtarget||target).push body if body
+          end
+          target.push ensure_.parsetree(o) if ensure_
+          target.push [:nil] if @empty_ensure
+          target=newtarget if newtarget
+
+          unless rescues.empty?
+            target.push linked_list(rescues)
+          end
+          target.push else_.parsetree(o) if  else_ #and !body
+          result.size==0 and result=[[:nil]]
+          result=result.last #if @op_rescue
+          result
+      end
+
+      def unparse_and_rescues(o)
+          result=" "
+          result+= body.unparse(o) if body
+          result+=unparse_nl(rescues.first,o)
+          rescues.each{|resc| result+=resc.unparse(o) }
+          result+=unparse_nl(else_,o)+"else "+else_.unparse(o) if else_
+          result+=";else" if @empty_else
+          result+=unparse_nl(ensure_,o)+"ensure "+ensure_.unparse(o) if ensure_
+          result+=";ensure" if @empty_ensure 
+          return result
+       end
+
+    end
+
+    class BeginNode<ValueNode
+      include HasRescue
+      param_names :body, :rescues, :else!, :ensure!
+      def initialize(*args)
+        @empty_ensure=@empty_else=@op_rescue=nil
+          body,rescues,else_,ensure_=*args[1...-1]
+          if else_
+            else_=else_.val or @empty_else=true
+          end
+          if ensure_
+            ensure_=ensure_.val or @empty_ensure=true
+          end
+          replace [body,rescues,else_,ensure_]
+      end
+
+      def op?; false end
+
+      alias ensure_ ensure
+      alias else_ else
+
+      attr_reader :empty_ensure, :empty_else
+      attr_accessor :after_comma, :after_equals
+
+      identity_param :after_equals, nil,  true
+
+      def image; '(begin)' end
+
+      def special_conditions!; nil end
+
+      def non_empty
+        body.size+rescues.size > 0 or !!ensures
+      end
+
+      identity_param :non_empty, false, true
+
+      def to_lisp
+        huh #what about rescues, else, ensure?
+        body.to_lisp
+      end
+ 
+      def parsetree(o)
+        result=parsetree_and_rescues(o)
+        result=[:begin,result] unless o[:ruby187]||result==[:nil]#||result.first==:begin
+        return result
+      end
+
+      def rescue_parsetree o
+        result=parsetree o
+        result.first==:begin and result=result.last unless o[:ruby187]
+        result
+      end
+
+      def begin_parsetree(o)
+        body,rescues,else_,ensure_=*self
+        needbegin=(rescues&&!rescues.empty?) || ensure_ || @empty_ensure
+        result=parsetree(o)
+        needbegin and result=[:begin, result] unless result.first==:begin
+        result
+      end
+
+      def lvalue
+        return nil
+      end
+#      attr_accessor :lvalue
+
+      def unparse(o=default_unparse_options)
+          result="begin "
+          result+=unparse_and_rescues(o)
+          result+=";end"
+      end
+    end
+
+    class RescueOpNode<ValueNode
+    #  include OpNode
+      param_names :body, :rescues #, :else!, :ensure!
+      def initialize(expr,rescueword,backup)
+            replace [expr,[RescueNode[[],nil,backup]]]
+      end
+
+      def else; nil end
+      def ensure; nil end
+
+      def left; body end
+      def right; rescues.action end
+
+      alias ensure_ ensure
+      alias else_ else
+      alias empty_ensure ensure
+      alias empty_else else
+
+      attr_accessor :after_equals
+      def op?; true end
+
+      def special_conditions!
+        nil
+      end
+
+      def to_lisp
+        huh #what about rescues
+        body.to_lisp
+      end
+ 
+      def parsetree(o)
+          body=body()
+          target=result=[]   #was: [:begin, ]
+
+          #body,rescues,else_,ensure_=*self
+
+          rescues=rescues().map{|resc| resc.parsetree(o)}
+          target.push newtarget=[:rescue, ]
+          else_=nil
+          needbegin=  (BeginNode===body and body.after_equals)
+          huh if needbegin and RescueOpNode===body #need test case for this
+          huh if needbegin and ParenedNode===body #need test case for this
+          body=body.parsetree(o) 
+          body=[:begin, body] if needbegin and body.first!=:begin and !o[:ruby187]
+          newtarget.push body if body
+          
+          newtarget.push linked_list(rescues)
+          result=result.last if result.size==1
+#          result=[:begin,result] 
+          result
+      end
+
+      def old_rescue_parsetree o
+        result=parsetree o
+        result=result.last unless o[:ruby187]
+        result
+      end
+
+      alias begin_parsetree parsetree
+      alias rescue_parsetree parsetree
+
+      def lvalue
+        return nil 
+      end
+
+      def unparse(o=default_unparse_options)
+          result= body.unparse(o)
+          result+=" rescue "
+          result+=rescues.first.action.unparse(o)
+      end
+    end
 
     class AssignmentRhsNode < Node #not to appear in final parse tree
       param_names :open_, :val, :close_
@@ -1296,6 +1914,19 @@ class RedParse
         else super args[1]
         end
       end
+      #WITHCOMMAS=UnaryStarNode|CommaOpNode|(CallSiteNode&-{:with_commas=>true})
+      def is_list
+
+        return !(WITHCOMMAS===val)
+=begin
+        #this should be equivalent, why doesn't it work?
+        !(UnaryStarNode===val or
+          CommaOpNode===val or
+          CallSiteNode===val && val.with_commas==true)
+#          CallSiteNode===val && !val.real_parens && val.args.size>0
+=end
+      end
+      identity_param :is_list, true, false
     end
 
     class AssignNode<ValueNode
@@ -1307,7 +1938,7 @@ class RedParse
         if args.size==5
           if args[3].ident=="rescue3"
             lhs,op,rescuee,op2,rescuer=*args
-            rhs=ParenedNode.new(rescuee.val,op2,rescuer)
+            rhs=RescueOpNode.new(rescuee.val,op2,rescuer)
           else
             lhs,op,bogus1,rhs,bogus2=*args
           end
@@ -1319,14 +1950,14 @@ class RedParse
         when UnaryStarNode         #look for star on lhs
           lhs=MultiAssign.new([lhs]) unless lhs.after_comma
         when ParenedNode
-        if lhs.size==1 and !lhs.after_comma      #look for () around lhs
+          if !lhs.after_comma      #look for () around lhs
           if CommaOpNode===lhs.first
             lhs=MultiAssign.new(Array.new(lhs.first))
           else
             lhs=MultiAssign.new([lhs.first])
           end
           @lhs_parens=true
-        end
+          end
         when CommaOpNode: 
           lhs=MultiAssign.new lhs
           #rhs=Array.new(rhs) if CommaOpNode===rhs
@@ -1361,10 +1992,12 @@ class RedParse
 
       def to_lisp
         case left
-        when ParenedNode: huh
-        when ConstantNode: huh
-        when BracketsGetNode: huh
-        when VarNameToken
+        when ParenedNode; huh
+        when BeginNode; huh
+        when RescueOpNode; huh
+        when ConstantNode; huh
+        when BracketsGetNode; huh
+        when VarNode
           "(set #{left.to_lisp} (#{op.chomp('=')} #{left.to_lisp} #{right.to_lisp}))"
         when CallSiteNode
         if op=='='
@@ -1385,26 +2018,28 @@ class RedParse
           left.all_current_lvars : []
       end
 
-      def parsetree
+      def parsetree(o)
         case left
-        when ParenedNode: huh
-        when ConstantNode: 
-        left.lvalue_parsetree << right.parsetree
-
-        when MultiAssign: 
-        lhs=left.lvalue_parsetree
-        rhs= right.class==Array ? right : [right]
+        when ParenedNode; huh
+        when RescueOpNode; huh
+        when BeginNode; huh
+        when ConstantNode;
+          left.lvalue_parsetree(o) << right.parsetree(o)
+
+        when MultiAssign;
+        lhs=left.lvalue_parsetree(o)
+        rhs= right.class==Array ? right.dup : [right]
         star=rhs.pop if UnaryStarNode===rhs.last
-        rhs=rhs.map{|x| x.rescue_parsetree}
+        rhs=rhs.map{|x| x.rescue_parsetree(o)}
         if rhs.size==0
           star or fail
-          rhs= star.parsetree
+          rhs= star.parsetree(o)
         elsif rhs.size==1 and !star and !(UnaryStarNode===left.first)
           rhs.unshift :to_ary
         else
           rhs.unshift(:array) 
           if star
-            splat=star.val.rescue_parsetree
+            splat=star.val.rescue_parsetree(o)
             #if splat.first==:call #I don't see how this can be right....
             #  splat[0]=:attrasgn
             #  splat[2]="#{splat[2]}=".to_sym
@@ -1413,15 +2048,23 @@ class RedParse
           end
           if left.size==1 and !(UnaryStarNode===left.first) and !(NestedAssign===left.first)
             rhs=[:svalue, rhs]
+            if CallNode===left.first
+              rhs=[:array, rhs]
+            end
           end
         end
-        lhs<< rhs
+        if left.size==1 and BracketsGetNode===left.first and right.class==Array #hack
+          lhs.last<<rhs
+          lhs
+        else
+          lhs<< rhs
+        end
 
         when CallSiteNode
         op=op().chomp('=')
-        rcvr=left.receiver.parsetree
+        rcvr=left.receiver.parsetree(o)
         prop=left.name.+('=').to_sym
-        args=right.rescue_parsetree
+        args=right.rescue_parsetree(o)
         UnaryStarNode===right and args=[:svalue, args]
         if op.empty?
           [:attrasgn, rcvr, prop, [:array, args] ]
@@ -1432,9 +2075,9 @@ class RedParse
         when BracketsGetNode
         args=left.params
         if op()=='='
-          result=left.lvalue_parsetree #[:attrasgn, left[0].parsetree, :[]=]
+          result=left.lvalue_parsetree(o) #[:attrasgn, left[0].parsetree(o), :[]=]
           result.size==3 and result.push [:array]
-          rhs=right.rescue_parsetree
+          rhs=right.rescue_parsetree(o)
           UnaryStarNode===right and rhs=[:svalue, rhs]
           if args 
             result[-1]=[:argspush,result[-1]] if UnaryStarNode===args.last
@@ -1445,7 +2088,7 @@ class RedParse
           
         else
 =begin
-          args&&=args.map{|x| x.parsetree}.unshift(:array)
+          args&&=args.map{|x| x.parsetree(o)}.unshift(:array)
           splat=args.pop if :splat==args.last.first
           if splat and left.params.size==1
             args=splat
@@ -1453,7 +2096,7 @@ class RedParse
             args=[:argscat, args, splat.last]
           end
 =end
-          lhs=left.parsetree
+          lhs=left.parsetree(o)
           if lhs.first==:fcall
             rcvr=[:self]
             args=lhs[2]
@@ -1465,37 +2108,37 @@ class RedParse
           result=[
             :op_asgn1, rcvr, args,
             op().chomp('=').to_sym, 
-            right.rescue_parsetree
+            right.rescue_parsetree(o)
           ]
         end
 
-        when VarNameToken
+        when VarNode
         node_type=left.varname2assigntype
         if /^(&&|\|\|)=$/===op()
           
           return ["op_asgn_#{$1[0]==?& ? "and" : "or"}".to_sym, 
-                  left.parsetree,
+                  left.parsetree(o),
                   [node_type, left.ident.to_sym, 
-                   right.rescue_parsetree]
+                   right.rescue_parsetree(o)]
                  ]
         end
 
         if op()=='='
-          rhs=right.rescue_parsetree
+          rhs=right.rescue_parsetree(o)
           UnaryStarNode===right and rhs=[:svalue, rhs]
           
 #          case left
-#          when VarNameToken: 
+#          when VarNode: 
                [node_type, left.ident.to_sym, rhs]
-#          else [node_type, left.data[0].parsetree, left.data[1].data[0].ident.+('=').to_sym  ,[:array, rhs]]
+#          else [node_type, left.data[0].parsetree(o), left.data[1].data[0].ident.+('=').to_sym  ,[:array, rhs]]
 #          end
 
 =begin these branches shouldn't be necessary now
         elsif node_type==:op_asgn2
-          [node_type, @data[0].data[0].parsetree, 
+          [node_type, @data[0].data[0].parsetree(o), 
             @data[0].data[1].data[0].ident.+('=').to_sym,
             op().ident.chomp('=').to_sym,
-             @data[2].parsetree
+             @data[2].parsetree(o)
           ]
         elsif node_type==:attrasgn
           [node_type]
@@ -1503,9 +2146,9 @@ class RedParse
         else
           [node_type, left.ident.to_sym,
             [:call,
-             left.parsetree, 
+             left.parsetree(o), 
                op().chomp('=').to_sym, 
-             [:array, right.rescue_parsetree]
+             [:array, right.rescue_parsetree(o)]
             ]
           ]
         end
@@ -1514,8 +2157,10 @@ class RedParse
         end
       end
 
-      def unparse(o)
-        lhs.lhs_unparse(o)+op+
+      def unparse(o=default_unparse_options)
+        result=lhs.lhs_unparse(o)
+        result="(#{result})" if defined? @lhs_parens
+        result+op+
           (rhs.class==Array ? 
             rhs.map{|rv| rv.unparse o}.join(',') :
             rhs.unparse(o) 
@@ -1528,18 +2173,18 @@ class RedParse
 
       #not called from parse table
 
-      def parsetree
+      def parsetree(o)
         lhs=left.dup
         if UnaryStarNode===lhs.last
           lstar=lhs.pop
         end
         lhs.map!{|x| 
-          res=x.parsetree
-          res[0]=x.varname2assigntype if VarNameToken===x
+          res=x.parsetree(o)
+          res[0]=x.varname2assigntype if VarNode===x
           res
         }
         lhs.unshift(:array) if lhs.size>1 or lstar
-        rhs=right.map{|x| x.parsetree}
+        rhs=right.map{|x| x.parsetree(o)}
         if rhs.size==1
           if rhs.first.first==:splat 
             rhs=rhs.first
@@ -1558,7 +2203,7 @@ class RedParse
           end
         end
         result=[:masgn, lhs, rhs]
-        result.insert(2,lstar.data.last.parsetree) if lstar
+        result.insert(2,lstar.data.last.parsetree(o)) if lstar
         result
 
       end
@@ -1567,11 +2212,11 @@ class RedParse
     class AssigneeList< ValueNode #abstract
       def initialize(data)
         data.each_with_index{|datum,i|
-          if ParenedNode&-{:size=>1}===datum
+          if ParenedNode===datum
             first=datum.first
             list=case first
-            when CommaOpNode: Array.new(first)
-            when UnaryStarNode,ParenedNode: [first]
+            when CommaOpNode; Array.new(first)
+            when UnaryStarNode,ParenedNode; [first]
             end 
             data[i]=NestedAssign.new(list) if list
           end
@@ -1579,18 +2224,18 @@ class RedParse
         replace data
       end
 
-      def  unparse o
+      def  unparse o=default_unparse_options
         map{|lval| lval.lhs_unparse o}.join(', ')      
       end
 
-      def old_parsetree
+      def old_parsetree o
         lhs=data.dup
         if UnaryStarNode===lhs.last
           lstar=lhs.pop.val
         end
         lhs.map!{|x| 
-          res=x.parsetree
-          res[0]=x.varname2assigntype if VarNameToken===x
+          res=x.parsetree(o)
+          res[0]=x.varname2assigntype if VarNode===x
           res
         }
         lhs.unshift(:array) if lhs.size>1 or lstar
@@ -1605,22 +2250,22 @@ class RedParse
 
       end
 
-      def parsetree 
+      def parsetree(o) 
         data=self
         data.empty? and return nil 
 #        data=data.first if data.size==1 and ParenedNode===data.first and data.first.size==1
         data=Array.new(data)
         star=data.pop if UnaryStarNode===data.last
-        result=data.map{|x| x.lvalue_parsetree }
+        result=data.map{|x| x.lvalue_parsetree(o) }
 =begin
         {
-          if VarNameToken===x 
+          if VarNode===x
             ident=x.ident
             ty=x.varname2assigntype
 #            ty==:lasgn and ty=:dasgn_curr
             [ty, ident.to_sym] 
           else
-            x=x.parsetree
+            x=x.parsetree(o)
             if x[0]==:call
               x[0]=:attrasgn
               x[2]="#{x[2]}=".to_sym
@@ -1629,14 +2274,17 @@ class RedParse
           end
         }
 =end
-        if result.size==0
+        if result.size==0   #just star on lhs
           star or fail
-          result=[:masgn, star.lvalue_parsetree]
-        elsif result.size==1 and !star and !(NestedAssign===data.first)
+          result=[:masgn]
+          result.push nil #why??? #if o[:ruby187]
+          result.push star.lvalue_parsetree(o)
+        elsif result.size==1 and !star and !(NestedAssign===data.first)  #simple lhs, not multi
           result=result.first
         else
           result=[:masgn, [:array, *result]]
-          result.push star.lvalue_parsetree if star and not DanglingCommaNode===star
+          result.push nil if (!star or DanglingCommaNode===star) #and o[:ruby187]
+          result.push star.lvalue_parsetree(o) if star and not DanglingCommaNode===star
         end
         result
       end
@@ -1652,13 +2300,18 @@ class RedParse
         return result
       end
 
-      def lvalue_parsetree; parsetree end
+      def lvalue_parsetree(o); parsetree(o) end
     end
     class NestedAssign<AssigneeList
-#      def parsetree
+      def parsetree(o)
+        result=super
+        result<<nil #why???!! #if o[:ruby187]
+        result
+      end
+#      def parsetree(o)
 #        [:masgn, *super]
 #      end
-      def unparse o
+      def unparse o=default_unparse_options
         "("+super+")"
       
       end
@@ -1670,7 +2323,7 @@ class RedParse
       def initialize(data)
         item=data.first if data.size==1
         #elide 1 layer of parens if present
-        if ParenedNode===item and item.size==1
+        if ParenedNode===item
           item=item.first
           data=CommaOpNode===item ? Array.new(item) : [item]
           @had_parens=true
@@ -1679,13 +2332,19 @@ class RedParse
         super(data)
       end
 
-      def unparse o
-        unless @had_parens
-          "|"+super+"|" 
-        else
+      def unparse o=default_unparse_options
+        if defined? @had_parens
           "|("+super+")|" 
+        else
+          "|"+super+"|" 
         end
       end
+
+      def parsetree o
+        result=super
+        result.push nil if UnaryStarNode===self.last || size>1 #and o[:ruby187]
+        result
+      end
     end
 
     class AccessorAssignNode < ValueNode #obsolete
@@ -1703,11 +2362,11 @@ class RedParse
         end
       end
 
-      def parsetree
+      def parsetree(o)
         op=op().ident.chomp('=')
-        rcvr=left.parsetree
+        rcvr=left.parsetree(o)
         prop=property.ident.<<(?=).to_sym
-        rhs=right.parsetree
+        rhs=right.parsetree(o)
         if op.empty?
           [:attrasgn, rcvr, prop, [:array, args] ]
         else
@@ -1717,7 +2376,7 @@ class RedParse
     end
  
     module KeywordOpNode
-      def unparse o
+      def unparse o=default_unparse_options
         [left.unparse(o),' ',op,' ',right.unparse(o)].to_s
       end
     end
@@ -1727,8 +2386,8 @@ class RedParse
       def initialize(left,op,right)
         @opmap=op[0,1]
         case op
-        when "&&": op="and"
-        when "||": op="or"
+        when "&&"; op="and"
+        when "||"; op="or"
         end
         #@reverse= op=="or"
         #@op=op
@@ -1756,7 +2415,7 @@ class RedParse
 
       #these 3 methods are defined in RawOpNode too, hence these
       #definitions are ignored. grrrrrrr.
-      def unparse o
+      def unparse o=default_unparse_options
         result=''
       
         each_with_index{|expr,i|
@@ -1774,14 +2433,14 @@ class RedParse
         method_missing(:right,*args,&block)
       end
 
-      def parsetree
+      def parsetree(o)
         result=[].replace(self).reverse
-        last=result.shift.begin_parsetree
+        last=result.shift.begin_parsetree(o)
         first=result.pop
         result=result.inject(last){|sum,x| 
-          [op.to_sym, x.begin_parsetree, sum]
+          [op.to_sym, x.begin_parsetree(o), sum]
         }
-        [op.to_sym, first.rescue_parsetree, result]
+        [op.to_sym, first.rescue_parsetree(o), result]
       end
 
       def special_conditions!
@@ -1802,16 +2461,16 @@ class RedParse
         @reverse=false
         @module=WhileOpNode
         @loop=true
-        @test_first= !( ParenedNode===val1 and val1.size != 1  )
+        @test_first= !( BeginNode===val1 )
         condition.special_conditions! if condition.respond_to? :special_conditions!
       end
 
       def while; condition end
       def do; consequent end
 
-      def parsetree
-        cond=condition.rescue_parsetree
-        body=consequent.parsetree
+      def parsetree(o)
+        cond=condition.rescue_parsetree(o)
+        body=consequent.parsetree(o)
         !@test_first and
           body.size == 2 and
             body.first == :begin and
@@ -1836,7 +2495,7 @@ class RedParse
         self[1]=op
         @reverse=true
         @loop=true
-        @test_first= !( ParenedNode===val1 and (val1.size != 1 )) 
+        @test_first= !( BeginNode===val1 ) 
         @module=UntilOpNode
         condition.special_conditions! if condition.respond_to? :special_conditions!
       end
@@ -1844,9 +2503,9 @@ class RedParse
       def while; negate condition end
       def do; consequent end
 
-      def parsetree
-        cond=condition.rescue_parsetree
-        body=consequent.parsetree
+      def parsetree(o)
+        cond=condition.rescue_parsetree(o)
+        body=consequent.parsetree(o)
         !@test_first and
           body.size == 2 and
             body.first == :begin and
@@ -1857,10 +2516,10 @@ class RedParse
         else
           kw=:until
         end
-        tf=@test_first
-        tf||= (!consequent.body and !consequent.else and !consequent.empty_else and
-               !consequent.ensure and !consequent.empty_ensure and consequent.rescues.empty?
-              ) if ParenedNode===consequent
+        tf=@test_first||body==[:nil]
+#        tf||= (!consequent.body and !consequent.else and #!consequent.empty_else and
+#               !consequent.ensure and !consequent.empty_ensure and consequent.rescues.empty?
+#              ) if BeginNode===consequent
         [kw, cond, body, tf]
       end
     end
@@ -1882,9 +2541,9 @@ class RedParse
       def else; consequent end
       def elsifs; [] end
 
-      def parsetree
-        cond=condition.rescue_parsetree
-        actions=[nil, consequent.parsetree]
+      def parsetree(o)
+        cond=condition.rescue_parsetree(o)
+        actions=[nil, consequent.parsetree(o)]
         if cond.first==:not
           actions.reverse!
           cond=cond.last
@@ -1910,9 +2569,9 @@ class RedParse
       def else; nil end
       def elsifs; [] end
 
-      def parsetree
-        cond=condition.rescue_parsetree
-        actions=[consequent.parsetree, nil]
+      def parsetree(o)
+        cond=condition.rescue_parsetree(o)
+        actions=[consequent.parsetree(o), nil]
         if cond.first==:not
           actions.reverse!
           cond=cond.last
@@ -1946,11 +2605,17 @@ class RedParse
           if first
             arrowrange=first..last
             arrows=param_list[arrowrange]
-            param_list[arrowrange]=[HashLiteralNode.new(nil,arrows,nil)]
+            h=HashLiteralNode.new(nil,arrows,nil)
+            h.startline=arrows.first.startline
+            h.endline=arrows.last.endline
+            param_list[arrowrange]=[h]
           end
         
-        when ArrowOpNode:  
-          param_list=[HashLiteralNode.new(nil,param_list,nil)]
+        when ArrowOpNode
+          h=HashLiteralNode.new(nil,param_list,nil)
+          h.startline=param_list.startline
+          h.endline=param_list.endline
+          param_list=[h]
 #        when KeywordOpNode
 #          fail "didn't expect '#{param_list.inspect}' inside actual parameter list"
         when nil
@@ -1972,12 +2637,12 @@ class RedParse
 
       def real_parens; !@not_real_parens end
 
-      def unparse o
+      def unparse o=default_unparse_options
         fail if block==false
         result=[
          receiver&&receiver.unparse(o)+'.',name,      
          real_parens ? '(' : (' ' if params),
-         params&&params.map{|param| param.unparse(o)}.join(', '),
+         params&&params.map{|param|  unparse_nl(param,o,'',"\\\n")+param.unparse(o)  }.join(', '),
          real_parens ? ')' : nil,
         
          block&&[
@@ -1994,19 +2659,27 @@ class RedParse
       def image
         result="(#{receiver.image if receiver}.#{name})"
       end
+     
+      def with_commas
+         !real_parens and args.size>0
+      end
+
+#      identity_param :with_commas, false, true
 
-      def lvalue_parsetree
-        result=parsetree
+      def lvalue_parsetree(o)
+        result=parsetree(o)
         result[0]=:attrasgn
         result[2]="#{result[2]}=".to_sym
         result
       end
 
-      def lvalue?
+      def lvalue
         return @lvalue if defined? @lvalue
         @lvalue=true
       end
-      attr_accessor :lvalue
+      attr_writer :lvalue
+
+      identity_param :lvalue, nil, true
 
       def to_lisp
         "(#{receiver.to_lisp} #{self[1..-1].map{|x| x.to_lisp}.join(' ')})"
@@ -2019,7 +2692,7 @@ class RedParse
         self[0]=expr
       end
 
-      def parsetree_with_params
+      def parsetree_with_params o
         args=args()||[]
         if (UnOpNode===args.last and args.last.ident=="&@")
           lasti=args.size-2
@@ -2038,26 +2711,30 @@ class RedParse
           ]
         elsif (UnaryStarNode===args[lasti])
           if lasti.zero?
-            [:fcall, methodsym, args.first.rescue_parsetree]
+            [:fcall, methodsym, args.first.rescue_parsetree(o)]
           else
             [:fcall, methodsym, 
              [:argscat, 
-               [:array, *args[0...lasti].map{|x| x.rescue_parsetree } ], 
-                  args[lasti].val.rescue_parsetree 
+               [:array, *args[0...lasti].map{|x| x.rescue_parsetree(o) } ], 
+                  args[lasti].val.rescue_parsetree(o) 
              ]
             ]
           end         
         else
           singlearg= lasti.zero?&&args.first
           [:fcall, methodsym, 
-           [:array, *args[0..lasti].map{|x| x.rescue_parsetree } ]
+           [:array, *args[0..lasti].map{|x| x.rescue_parsetree(o) } ]
           ]
         end
 
         result[0]=:vcall if block #and /\Af?call\Z/===result[0].to_s
 
         if is_kw and !receiver
-          return [methodsym, singlearg.parsetree] if singlearg and "super"!=methodname
+          if singlearg and "super"!=methodname
+            result=[methodsym, singlearg.parsetree(o)]
+            result.push(true) if methodname=="yield" and ArrayLiteralNode===singlearg #why???!!
+            return result
+          end
           breaklike=  /^(break|next|return)$/===methodname
           if @not_real_parens
               return [:zsuper] if "super"==methodname and !args()
@@ -2071,31 +2748,31 @@ class RedParse
 
         if receiver
           result.shift if result.first==:vcall or result.first==:fcall #if not kw
-          result=[:call, receiver.rescue_parsetree, *result]
+          result=[:call, receiver.rescue_parsetree(o), *result]
         end
 
         if unamp_expr
 #          result[0]=:fcall if lasti.zero?
-          result=[:block_pass, unamp_expr.rescue_parsetree, result]
+          result=[:block_pass, unamp_expr.rescue_parsetree(o), result]
         end
 
         return result
       end
 
-      def parsetree
-        callsite=parsetree_with_params
+      def parsetree(o)
+        callsite=parsetree_with_params o
         return callsite unless blockparams or block
         call=name
         callsite[0]=:fcall  if callsite[0]==:call or callsite[0]==:vcall
         unless receiver
           case call
-          when "BEGIN": 
-            if quirks
+          when "BEGIN"
+            if o[:quirks]
               return []
             else
               callsite=[:preexe] 
             end
-          when "END": callsite=[:postexe]
+          when "END"; callsite=[:postexe]
           end
         else
           callsite[0]=:call if callsite[0]==:fcall
@@ -2105,17 +2782,17 @@ class RedParse
           bparams=blockparams.dup
           lastparam=bparams.last
           amped=bparams.pop.val if UnOpNode===lastparam and lastparam.op=="&@"
-          bparams=bparams.parsetree||0
+          bparams=bparams.parsetree(o)||0
           if amped
             bparams=[:masgn, [:array, bparams]] unless bparams==0 or bparams.first==:masgn
-            bparams=[:block_pass, amped.lvalue_parsetree, bparams]
+            bparams=[:block_pass, amped.lvalue_parsetree(o), bparams]
           end
         else
           bparams=nil
         end
         result=[:iter, callsite, bparams]
         unless block.empty?
-          body=block.parsetree
+          body=block.parsetree(o)
           if curr_vars=block.lvars_defined_in
             curr_vars-=blockparams.all_current_lvars if blockparams
             if curr_vars.empty?
@@ -2133,19 +2810,19 @@ class RedParse
         result
       end
 
-      def blockformals_parsetree data
+      def blockformals_parsetree data,o  #dead code?
         data.empty? and return nil 
         data=data.dup
         star=data.pop if UnaryStarNode===data.last
-        result=data.map{|x| x.parsetree }
+        result=data.map{|x| x.parsetree(o) }
 =begin
- {          if VarNameToken===x 
+ {          if VarNode===x
             ident=x.ident
             ty=x.varname2assigntype
 #            ty==:lasgn and ty=:dasgn_curr
             [ty, ident.to_sym] 
           else
-            x=x.parsetree
+            x=x.parsetree(o)
             if x[0]==:call
               x[0]=:attrasgn
               x[2]="#{x[2]}=".to_sym
@@ -2156,20 +2833,20 @@ class RedParse
 =end
         if result.size==0
           star or fail
-          result=[:masgn, star.parsetree.last]
+          result=[:masgn, star.parsetree(o).last]
         elsif result.size==1 and !star
           result=result.first
         else
           result=[:masgn, [:array, *result]]
           if star 
             old=star= star.val
-            star=star.parsetree
+            star=star.parsetree(o)
             if star[0]==:call
               star[0]=:attrasgn
               star[2]="#{star[2]}=".to_sym
             end
 
-            if VarNameToken===old
+            if VarNode===old
               ty=old.varname2assigntype
 #              ty==:lasgn and ty=:dasgn_curr
               star[0]=ty
@@ -2205,11 +2882,11 @@ class RedParse
         "(#{data.join' '})"
       end
 
-      def parsetree
+      def parsetree(o)
         empty? ? nil :
           [:dasgn_curr, 
             *map{|x| 
-              VarNameToken===x ? x.ident.to_sym : x.parsetree
+              (VarNode===x) ? x.ident.to_sym : x.parsetree(o)
             }
           ]
       end
@@ -2219,8 +2896,8 @@ class RedParse
       param_names :params,:body
       def initialize(open_brace,formals,stmts,close_brace)
         case stmts
-        when SequenceNode: stmts=Array.new(stmts)
-        when nil: stmts=[]
+        when SequenceNode; stmts=Array.new(stmts)
+        when nil; stmts=[]
         else stmts=[stmts]
         end
         
@@ -2235,23 +2912,23 @@ class RedParse
         "(#{params.to_lisp} #{body.to_lisp})"
       end
 
-      def parsetree #obsolete
-        callsite=@data[0].parsetree
+      def parsetree(o) #obsolete
+        callsite=@data[0].parsetree(o)
         call=@data[0].data[0]
         callsite[0]=:fcall  if call.respond_to? :ident
         if call.respond_to? :ident
           case call.ident
-          when "BEGIN": 
-            if quirks
+          when "BEGIN" 
+            if o[:quirks]
               return []
             else
               callsite=[:preexe] 
             end
-          when "END": callsite=[:postexe]
+          when "END"; callsite=[:postexe]
           end
         end
-        result=[:iter, callsite, @data[1].parsetree]
-        result.push @data[2].parsetree if @data[2]
+        result=[:iter, callsite, @data[1].parsetree(o)]
+        result.push @data[2].parsetree(o) if @data[2]
         result
       end
     end
@@ -2261,7 +2938,7 @@ class RedParse
         super()
       end
 
-      def unparse o
+      def unparse o=default_unparse_options
         ''      
       end
 
@@ -2285,7 +2962,7 @@ class RedParse
         "Object"
       end
 
-      def parsetree
+      def parsetree(o)
         :Object
       end
     end
@@ -2313,15 +2990,25 @@ class RedParse
           @char=str.char
         end
         @modifiers=str.modifiers #if str.modifiers
-        super *with_string_data(str)
+        super( *with_string_data(str) )
+
+        @open=token.open
+        @close=token.close
+        @bs_handler=str.bs_handler
 
         if /[\[{]/===@char
           @parses_like=split_into_words(str)
         end
 
+        return
+
+=begin
+#this should have been taken care of by with_string_data        
         first=shift
         delete_if{|x| ''==x }
         unshift(first)
+
+#escape translation now done later on
         map!{|strfrag|
           if String===strfrag
             str.translate_escapes strfrag
@@ -2329,8 +3016,25 @@ class RedParse
             strfrag
           end
         }
-        @open=token.open
-        @close=token.close
+=end
+      end
+
+      def translate_escapes(str)
+        rl=RubyLexer.new("(string escape translation hack...)",'')
+        result=str.dup
+        seq=result.to_sequence
+        rl.instance_eval{@file=seq}
+        repls=[]
+        i=0
+        #ugly ugly ugly... all so I can call @bs_handler
+        while i<result.size and bs_at=result.index(/\\./m,i)
+          seq.pos=$~.end(0)-1
+          ch=rl.send(@bs_handler,"\\",@open[-1,1],@close)
+          result[bs_at...seq.pos]=ch
+          i=bs_at+ch.size
+        end
+
+        return  result
       end
 
       def old_cat_initialize(*tokens) #not needed anymore?
@@ -2359,45 +3063,59 @@ class RedParse
 
       ESCAPABLES={}
       EVEN_NUM_BSLASHES=/(^|[^\\])((?:\\\\)*)/
-      def unparse o
-      
-        [@open,unparse_interior(o),@close,@modifiers].to_s  
+      def unparse o=default_unparse_options
+        o[:linenum]+=@open.count("\n")
+        result=[@open,unparse_interior(o),@close,@modifiers].to_s  
+        o[:linenum]+=@close.count("\n")
+        return result
       end
 
       def escapable open=@open,close=@close
         unless escapable=ESCAPABLES[open]
-          maybe_crunch='#' if %r{\A["`/\{]\Z} === @char #"
+          maybe_crunch='\\#' if %r{\A["`/\{]\Z} === @char and open[1] != ?q and open != "'" #"
           #crunch (#) might need to be escaped too, depending on what @char is
           escapable=ESCAPABLES[open]=
-            /[#{open[-1,1]+close}#{maybe_crunch}]/
+            /[#{Regexp.quote open[-1,1]+close}#{maybe_crunch}]/
         end
         escapable             
       end
 
-      def unparse_interior o,open=@open,close=@close
+      def unparse_interior o,open=@open,close=@close,escape=nil
         escapable=escapable(open,close)
-        map{|substr|
+        result=map{|substr|
           if String===substr
-            substr.gsub! /\\+/ do 
-              result=$&*2 
-              result.chomp! '\\' if $&.size&1==1
-              result
-            end
-            substr.gsub! escapable do '\\'+$& end
+
+            #hack: this is needed for here documents only, because their
+            #delimiter is changing.
+            substr.gsub!(escape){|ch| ch[0...-1]+"\\"+ch[-1,1]} if escape
+
+            o[:linenum]+=substr.count("\n") if o[:linenum]
+
             substr
           else
             ['#{',substr.unparse(o),'}']
           end
         }
+        result
       end
 
       def image; '(#@char)' end
 
       def delete_extraneous_ivars!
-        @parses_like.delete_extraneous_ivars! if @parses_like
+        @parses_like.delete_extraneous_ivars! if defined? @parses_like
         return super
       end
 
+      def walk(*args,&callback)
+        return @parses_like.walk(*args,&callback) if defined? @parses_like
+        super
+      end
+
+      def depthwalk(*args,&callback)
+        return @parses_like.depthwalk(*args,&callback) if defined? @parses_like
+        super
+      end
+
       def special_conditions!
         @implicit_match= @char=="/"
       end
@@ -2412,45 +3130,57 @@ class RedParse
 #          data=elems=token.string.elems
           data=elems=
             case token
-            when StringToken: token.elems
-            when HerePlaceholderToken: token.string.elems
+            when StringToken; token.elems
+            when HerePlaceholderToken; token.string.elems
             else raise "unknown string token type: #{token}:#{token.class}"
             end
 #          sum.size%2==1 and sum.last<<elems.shift
 #          sum+elems
 #        } 
+#        endline=@endline
         1.step(data.length-1,2){|i|
           tokens=data[i].ident.dup
+          line=data[i].linenum
+
+          #replace trailing } with EoiToken
           (tokens.size-1).downto(0){|j| 
              tok=tokens[j]
              break(tokens[j..-1]=[EoiToken.new('',nil,tokens[j].offset)]) if tok.ident=='}' 
           }
+          #remove leading {
           tokens.each_with_index{|tok,j| break(tokens.delete_at j) if tok.ident=='{' }
+
           if tokens.size==1 and VarNameToken===tokens.first
-            data[i]=tokens.first
+            data[i]=VarNode.new tokens.first
+            data[i].endline=token.endline
           else
-            klass=Thread.current[:$RedParse_parser].class
-            data[i]=klass.new(tokens, "(string inclusion)").parse
+            #parse the token list in the string inclusion
+            parser=Thread.current[:$RedParse_parser]
+            klass=parser.class
+            data[i]=klass.new(tokens, "(string inclusion)",1,[],{:rubyversion=>parser.rubyversion}).parse
           end
         } #if data
-#        was_nul_header= (String===data.first and data.first.empty?) #and quirks
+#        was_nul_header= (String===data.first and data.first.empty?) #and o[:quirks]
         last=data.size-1
+
+        #remove (most) empty string fragments
         last.downto(1){|frag_i| 
           frag=data[frag_i]
           String===frag or next
-          delete= frag.empty? 
-          next if frag_i==last #and quirks
-          next if data[frag_i-1].line != data[frag_i+1].line #and quirks
+          next unless frag.empty? 
+          next if frag_i==last #and o[:quirks]
+          next if data[frag_i-1].endline != data[frag_i+1].endline #and o[:quirks]
                   #prev and next inclusions on different lines
-          data.slice!(frag_i) if delete
+          data.slice!(frag_i)
         }
 #        data.unshift '' if was_nul_header
+
         return data
       end
 
-      def line= line
+      def endline= endline
         each{|frag| 
-          frag.line||=line if frag.respond_to? :line
+          frag.endline||=endline if frag.respond_to? :endline
         }
 
         super
@@ -2461,18 +3191,63 @@ class RedParse
         huh
       end
 
+      EVEN_BSS=/(?:[^\\\s\v]|\G)(?:\\\\)*/
+
+      DQ_ESC=/(?>\\(?>[CM]-|c)?)/
+      DQ_EVEN=%r[
+                  (?:
+                   \A |
+                   [^\\c-] |
+                   (?>\A|[^\\])c |
+                   (?> [^CM] | (?>\A|[^\\])[CM] )-
+                  )              #not esc
+                  #{DQ_ESC}{2}*  #an even number of esc
+             ]omx
+      DQ_ODD=/#{DQ_EVEN}#{DQ_ESC}/omx
+      SQ_ESC=/\\/
+      SQ_EVEN=%r[
+                  (?:  \A | [^\\]  )  #not esc
+                  #{SQ_ESC}{2}*       #an even number of esc
+             ]omx
+      SQ_ODD=/#{SQ_EVEN}#{SQ_ESC}/omx
       def split_into_words strtok
         return unless /[{\[]/===@char
         result=ArrayLiteralNode[]
-        result << StringNode['',{:@char=>'"'}]
-        first[/\A(?:\s|\v)+/]='' if /\A(?:\s|\v)/===first
-        each{|x|
+        result << StringNode['',{:@char=>'"',:@open=>@open,:@close=>@close,:@bs_handler=>@bs_handler}]
+        proxy=dup
+        proxy[0]=proxy[0][/\A(?:\s|\v)+(.*)\Z/m,1] if /\A(?:\s|\v)/===proxy[0]
+#        first[/\A(?:\s|\v)+/]='' if /\A(?:\s|\v)/===first #uh-oh, changes first
+        proxy.each{|x|
           if String===x
-            double_chunks=x.split(/((?:(?:[^\\]|\A)(?:\\\\)+)|(?:[^\\\s\v]|\A))(?:\s|\v)+/,-1)
+#            x=x[/\A(?:\s|\v)+(.*)\Z/,1] if /\A[\s\v]/===x
+if false
+            #split on ws preceded by an even # of backslashes or a non-backslash, non-ws char
+            #this ignores backslashed ws
+            #save the thing that preceded the ws, it goes back on the token preceding split
+            double_chunks=x.split(/( #{EVEN_BSS} | (?:[^\\\s\v]|\A|#{EVEN_BSS}\\[\s\v]) )(?:\s|\v)+/xo,-1)
             chunks=[]
             (0..double_chunks.size).step(2){|i| 
-              chunks << strtok.translate_escapes(double_chunks[i,2].to_s)#.gsub(/\\([\s\v\\])/){$1}
+              chunks << #strtok.translate_escapes \
+                double_chunks[i,2].to_s #.gsub(/\\([\s\v\\])/){$1}
             }
+else
+            #split on ws, then ignore ws preceded by an odd number of esc's
+            #esc is \ in squote word array, \ or \c or \C- or \M- in dquote
+            chunks_and_ws=x.split(/([\s\v]+)/,-1)
+            start=chunks_and_ws.size; start-=1 if start&1==1
+            chunks=[]
+            i=start+2; 
+            while (i-=2)>=0 
+              ch=chunks_and_ws[i]||""
+              if i<chunks_and_ws.size and ch.match(@char=="[" ? /#{SQ_ODD}\Z/omx : /#{DQ_ODD}\Z/omx)
+                ch<< chunks_and_ws[i+1][0,1]
+                if chunks_and_ws[i+1].size==1
+                  ch<< chunks.shift
+                end
+              end
+              chunks.unshift ch
+            end
+end
 
             chunk1= chunks.shift          
             if chunk1.empty?
@@ -2483,9 +3258,16 @@ class RedParse
               result.last.push chunk1
             end
 #            result.last.last.empty? and result.last.pop
-            result.concat chunks.map{|chunk| StringNode[chunk,{:@char=>'"'}]}
+            result.concat chunks.map{|chunk| 
+              StringNode[chunk,{:@char=>'"',:@open=>@open,:@close=>@close,:@bs_handler=>@bs_handler}]
+            }
           else
-            result.last << x
+            #result.last << x
+            unless String===result.last.last
+              result.push StringNode["",{:@char=>'"',:@open=>@open,:@close=>@close,:@bs_handler=>@bs_handler}]
+            end
+            result.last.push x
+#            result.push StringNode["",x,{:@char=>'"',:@open=>@open,:@close=>@close,:@bs_handler=>@bs_handler}]
           end
         } 
         result.shift if StringNode&-{:size=>1, :first=>''}===result.first
@@ -2514,9 +3296,9 @@ class RedParse
         :dstr=>:str,
         :dxstr=>:xstr,
       }
-      def parsetree
+      def parsetree(o)
         if size==1
-          val=first
+          val=translate_escapes first
           type=case @char
                when '"',"'"; :str
                when '/'
@@ -2535,8 +3317,8 @@ class RedParse
                  val=Regexp.new val,numopts|charset
                  :lit
                when '[','{'
-                 return @parses_like.parsetree
-
+                 return @parses_like.parsetree(o)
+=begin
                  double_chunks=val.split(/([^\\]|\A)(?:\s|\v)/,-1)
                  chunks=[]
                  (0..double_chunks.size).step(2){|i| 
@@ -2550,6 +3332,7 @@ class RedParse
                  words.pop if words.last.empty? unless words.empty?
                  return [:zarray] if words.empty? 
                  return words.map{|word| [:str,word]}.unshift(:array)
+=end
                when '`'; :xstr
                else raise "dunno what to do with #@char<StringToken"
                end
@@ -2559,7 +3342,8 @@ class RedParse
           vals=[]
           each{|elem| 
             case elem
-            when String: 
+            when String
+              elem=translate_escapes elem
               if saw_string
                 result=[:str, elem]
               else
@@ -2567,10 +3351,10 @@ class RedParse
                 result=elem
               end
               vals.push result
-            when NopNode:
+            when NopNode
               vals.push [:evstr]
-            when Node,VarNameToken:
-              res=elem.parsetree
+            when Node #,VarNameToken
+              res=elem.parsetree(o)
               if res.first==:str and @char != '{'
                 vals.push res
               elsif res.first==:dstr and @char != '{'
@@ -2582,7 +3366,7 @@ class RedParse
             end
           }
           while vals.size>1 and vals[1].first==:str
-            vals[0]<<vals.delete_at(1).last
+            vals[0]+=vals.delete_at(1).last
           end
           #vals.pop if vals.last==[:str, ""]
 
@@ -2603,8 +3387,9 @@ class RedParse
                  vals.push numopts|charset unless numopts|charset==0
                  val=/#{val}/
                  type
-               when '{': 
-                 return @parses_like.parsetree
+               when '{'
+                 return @parses_like.parsetree(o)
+=begin
                  vals[0]=vals[0].sub(/\A(\s|\v)+/,'') if /\A(\s|\v)/===vals.first
                  merged=Array.new(vals)
                  result=[]
@@ -2620,7 +3405,7 @@ class RedParse
                      if !result.empty? and frag=words.shift and !frag.last.empty?
                        result[-1]+=frag
                      end
-                     result.push *words
+                     result.push( *words )
                    else
                      result.push [:str,""] if result.empty?
                      if i.first==:evstr and i.size>1 and i.last.first==:str
@@ -2637,6 +3422,7 @@ class RedParse
                    end
                  }
                  return result.unshift(:array)
+=end
                when '`'; :dxstr
                else raise "dunno what to do with #@char<StringToken"
                end
@@ -2657,6 +3443,7 @@ class RedParse
       def initialize(token)
         token.node=self
         super(token)
+        @startline=token.string.startline
       end
       attr_accessor :list_to_append
 #      attr :token
@@ -2666,18 +3453,35 @@ class RedParse
         @char=token.quote
         if @list_to_append
           size%2==1 and token << @list_to_append.shift
-          push *@list_to_append
+          push( *@list_to_append )
           remove_instance_variable :@list_to_append
         end
       end
 
+      def translate_escapes x
+        return x if @char=="'"
+        super
+      end
+
 
       def flattened_ivars_equal?(other)
         StringNode===other
       end
 
-      def unparse o
-        [@char,(unparse_interior o,@char,@char),@char].to_s
+      def unparse o=default_unparse_options
+        lead=unparse_nl(self,o,'',"\\\n")
+        inner=unparse_interior o,@char,@char,
+                case @char
+                when "'" #single-quoted here doc is a special case; 
+                         #\ and ' are not special within it
+                         #(and therefore always escaped if converted to normal squote str)
+                         /['\\]/ 
+                when '"'; /#{DQ_EVEN}"/
+                when "`"; /#{DQ_EVEN}`/
+                else fail
+                end 
+        
+        [lead,@char, inner, @char].to_s
       end
     end
 
@@ -2688,22 +3492,16 @@ class RedParse
         @offset=old_val.offset
         val=old_val.ident
         case old_val
-        when SymbolToken: 
+        when SymbolToken
           case val[1]
-          when ?': #'
+          when ?' #'
             assert !old_val.raw.has_str_inc?
             val=old_val.raw.translate_escapes(old_val.raw.elems.first).to_sym
-            #val=eval(val[1..-1]).to_sym
-            #eval here is cheating!!!!
-            #too lazy to do it right just now
-          when ?": #"
+          when ?" #"
             if old_val.raw.has_str_inc?
               val=StringNode.new(old_val.raw) #ugly hack: this isn't literal
             else
               val=old_val.raw.translate_escapes(old_val.raw.elems.first).to_sym
-              #val=eval(val[1..-1]).to_sym
-              #eval here is cheating!!!!
-              #too lazy to do it right just now
             end
           else #val=val[1..-1].to_sym
             if StringToken===old_val.raw
@@ -2712,16 +3510,22 @@ class RedParse
               val=old_val.raw.to_sym
             end
           end
-        when NumberToken: 
+        when NumberToken 
           case val
-          when /\A-?0([^.]|\Z)/: val=val.oct
-          when /[.e]/i: val=val.to_f
+          when /\A-?0([^.]|\Z)/; val=val.oct
+          when /[.e]/i; val=val.to_f
           else val=val.to_i
           end
         end
         super(val)
       end
 
+      def bare_method
+        Symbol===val || StringNode===val
+      end
+
+      identity_param :bare_method, nil, false, true
+
       def image; "(#{':' if Symbol===val}#{val})" end
 
       def to_lisp
@@ -2731,18 +3535,18 @@ class RedParse
       Inf="999999999999999999999999999999999.9e999999999999999999999999999999999999"
       Nan="****shouldnt ever happen****"
 
-      def unparse o
+      def unparse o=default_unparse_options
         val=val()
         case val 
         when StringNode #ugly hack
           ":"+
             val.unparse(o)
         when Float
-          s= "%#{Float::DIG}.#{Float::DIG}f"%val
+          s= "%#{Float::DIG+1}.#{Float::DIG+1}f"%val
           case s
-          when /-inf/i: s="-"+Inf
-          when /inf/i:  s=    Inf
-          when /nan/i:  s=    Nan
+          when /-inf/i; s="-"+Inf
+          when /inf/i;  s=    Inf
+          when /nan/i;  s=    Nan
           else
             fail unless [s.to_f].pack("d")==[val].pack("d")
           end
@@ -2751,11 +3555,11 @@ class RedParse
         end
       end
 
-      def parsetree
+      def parsetree(o)
         val=val()
         case val 
         when StringNode #ugly hack
-          result= val.parsetree
+          result= val.parsetree(o)
           result[0]=:dsym
           return result
 =begin
@@ -2786,6 +3590,7 @@ class RedParse
         if name.ident=='(' 
           #simulate nil
           replace ['nil']
+          @value=false
         else
           replace [name.ident]
           @value=name.respond_to?(:value) && name.value
@@ -2800,12 +3605,12 @@ class RedParse
         name
       end
 
-      def unparse o
+      def unparse o=default_unparse_options
         name      
       end
 
-      def parsetree
-        if @value
+      def parsetree(o)
+        if (defined? @value) and @value
           type=:lit
           val=@value
           if name=="__FILE__"
@@ -2823,7 +3628,7 @@ class RedParse
       def initialize(lbrack,contents,rbrack)
         contents or return super()
         if CommaOpNode===contents
-          super *contents
+          super( *contents )
         else
           super contents
         end
@@ -2831,13 +3636,13 @@ class RedParse
 
       def image; "([])" end
 
-      def unparse o
-        "["+map{|item| item.unparse o}.join(', ')+"]"
+      def unparse o=default_unparse_options
+        "["+map{|item| unparse_nl(item,o,'')+item.unparse(o)}.join(', ')+"]"
       end
 
-      def parsetree
+      def parsetree(o)
         size.zero? and return [:zarray]
-        normals,star,amp=param_list_parse(self)
+        normals,star,amp=param_list_parse(self,o)
         result=normals.unshift :array
         if star
           if size==1
@@ -2854,9 +3659,9 @@ class RedParse
 
     class BracketsSetNode < ValueNode #obsolete
       param_names :left,:assign_,:right
-      def parsetree
-        [:attrasgn, left.data[0].parsetree, :[]=, 
-         [:array]+Array(left.data[1]).map{|x| x.parsetree}<< right.parsetree
+      def parsetree(o)
+        [:attrasgn, left.data[0].parsetree(o), :[]=, 
+         [:array]+Array(left.data[1]).map{|x| x.parsetree(o)}<< right.parsetree(o)
         ]
       end
     end
@@ -2867,11 +3672,11 @@ class RedParse
         super
       end
 
-      def parsetree
+      def parsetree(o)
         bracketargs=@data[0].data[1]
-        bracketargs=bracketargs ? bracketargs.map{|x| x.parsetree}.unshift(:array) : [:zarray]
-        [:op_asgn1, @data[0].data[0].parsetree, bracketargs, 
-         data[1].ident.chomp('=').to_sym, data[2].parsetree]
+        bracketargs=bracketargs ? bracketargs.map{|x| x.parsetree(o)}.unshift(:array) : [:zarray]
+        [:op_asgn1, @data[0].data[0].parsetree(o), bracketargs, 
+         data[1].ident.chomp('=').to_sym, data[2].parsetree(o)]
       end
     end
 
@@ -2898,13 +3703,14 @@ class RedParse
 
       attr_reader :empty_else
 
-      def unparse o
+      def unparse o=default_unparse_options
         result=@reverse ? "unless " : "if "
-        result+="#{condition.unparse o}\n"
-        result+="#{consequent.unparse(o)}\n" if consequent
-        result+=elsifs.map{|n| n.unparse(o)}.to_s if elsifs
-        result+="else "+else_.unparse(o)+"\n" if else_
-        result+="end"
+        result+="#{condition.unparse o}"
+        result+=unparse_nl(consequent,o)+"#{consequent.unparse(o)}" if consequent
+        result+=unparse_nl(elsifs.first,o)+elsifs.map{|n| n.unparse(o)}.to_s if elsifs
+        result+=unparse_nl(else_,o)+"else "+else_.unparse(o) if else_
+        result+=";else " if defined? @empty_else
+        result+=";end"
         return result
       end
 
@@ -2938,14 +3744,14 @@ class RedParse
         end
       end
  
-      def parsetree
-        elsepart=otherwise.parsetree if otherwise
+      def parsetree(o)
+        elsepart=otherwise.parsetree(o) if otherwise
         elsifs.reverse_each{|elsifnode|
-          elsepart=elsifnode.parsetree << elsepart
+          elsepart=elsifnode.parsetree(o) << elsepart
         }
-        cond=condition.rescue_parsetree
+        cond=condition.rescue_parsetree(o)
         actions=[
-              consequent&&consequent.parsetree, 
+              consequent&&consequent.parsetree(o), 
               elsepart
         ]
         if cond.first==:not
@@ -2975,8 +3781,8 @@ class RedParse
       param_names :ensureword_, :val
       alias body val
       def image; "(ensure)" end
-      def parsetree #obsolete?
-        (body=body()) ? body.parsetree : [:nil]
+      def parsetree(o) #obsolete?
+        (body=body()) ? body.parsetree(o) : [:nil]
       end
     end
 
@@ -2993,16 +3799,16 @@ class RedParse
 
       def image; "(elsif)" end
 
-      def unparse o
-        "elsif #{condition.unparse o}\n#{consequent.unparse o}\n"
+      def unparse o=default_unparse_options
+        "elsif #{condition.unparse o}#{unparse_nl(consequent,o)}#{consequent.unparse o};"
       end
 
       def to_lisp
         "("+condition.to_lisp+" "+consequent.to_lisp+")"
       end
 
-      def parsetree #obsolete?
-        [:if, condition.rescue_parsetree, consequent&&consequent.parsetree, ]
+      def parsetree(o) #obsolete?
+        [:if, condition.rescue_parsetree(o), consequent&&consequent.parsetree(o), ]
       end
     end
 
@@ -3020,11 +3826,11 @@ class RedParse
 
       def image; "(#{loopword})" end
 
-      def unparse o
+      def unparse o=default_unparse_options
         [@reverse? "until " : "while ",
-         condition.unparse(o), "\n",
+         condition.unparse(o), unparse_nl(body||self,o),
          body&&body.unparse(o),
-         "\nend"
+         ";end"
         ].to_s
       end
 
@@ -3041,15 +3847,15 @@ class RedParse
         "(#{@reverse ? :until : :while} #{condition.to_lisp}\n#{body.to_lisp})"
       end
 
-      def parsetree
-        cond=condition.rescue_parsetree
+      def parsetree(o)
+        cond=condition.rescue_parsetree(o)
         if cond.first==:not
           reverse=!@reverse
           cond=cond.last
         else
           reverse=@reverse
         end 
-        [reverse ? :until : :while, cond, body&&body.parsetree, true]
+        [reverse ? :until : :while, cond, body&&body.parsetree(o), true]
       end
     end
 
@@ -3067,12 +3873,12 @@ class RedParse
 
       attr_reader :empty_else
 
-      def unparse o
-        result="case #{condition&&condition.unparse(o)}\n"+
+      def unparse o=default_unparse_options
+        result="case #{condition&&condition.unparse(o)}"+
                whens.map{|wh| wh.unparse o}.to_s
 
-        result += "else "+otherwise.unparse(o)+"\n" if otherwise
-        result += "end"
+        result += unparse_nl(otherwise,o)+"else "+otherwise.unparse(o) if otherwise
+        result += ";end"
 
         return result
       end
@@ -3086,10 +3892,10 @@ class RedParse
         "\n)"
       end
   
-      def parsetree
-        [:case, condition&&condition.parsetree]+ 
-           whens.map{|whennode| whennode.parsetree}+
-        [otherwise&&otherwise.parsetree]
+      def parsetree(o)
+        [:case, condition&&condition.parsetree(o)]+ 
+           whens.map{|whennode| whennode.parsetree(o)}+
+        [otherwise&&otherwise.parsetree(o)]
       end
     end
 
@@ -3105,13 +3911,12 @@ class RedParse
 
       def image; "(when)" end
 
-      def unparse o
-        result="when "
+      def unparse o=default_unparse_options
+        result=unparse_nl(self,o)+"when "
         result+=condition.class==Array ? 
                   condition.map{|cond| cond.unparse(o)}.join(',') : 
                   condition.unparse(o) 
-        result+="\n"
-        result+=consequent.unparse(o)+"\n" if consequent
+        result+=unparse_nl(consequent,o)+consequent.unparse(o) if consequent
         result
       end
 
@@ -3127,19 +3932,19 @@ class RedParse
 
       end
 
-      def parsetree
+      def parsetree(o)
         conds=
           if Node|Token===condition
-            [condition.rescue_parsetree]
+            [condition.rescue_parsetree(o)]
           else
-            condition.map{|cond| cond.rescue_parsetree}
+            condition.map{|cond| cond.rescue_parsetree(o)}
           end
         if conds.last[0]==:splat
           conds.last[0]=:when
           conds.last.push nil
         end
         [:when, [:array, *conds],
-          consequent&&consequent.parsetree
+          consequent&&consequent.parsetree(o)
         ]
       end
     end
@@ -3148,7 +3953,7 @@ class RedParse
       param_names(:forword_,:for!,:inword_,:in!,:doword_,:do!,:endword_)
       def initialize(forword,for_,inword,in_,doword,do_,endword)
         #elide 1 layer of parens if present
-        for_=for_.first if ParenedNode===for_ and for_.size==1
+        for_=for_.first if ParenedNode===for_
         for_=CommaOpNode===for_ ? Array.new(for_) : [for_]
         super(BlockParams.new(for_),in_,do_)
       end
@@ -3159,18 +3964,17 @@ class RedParse
 
       def image; "(for)" end
 
-      def unparse o
-        "
-         for #{iterator.lhs_unparse(o)[1...-1]} in #{enumerable.unparse o}
-           #{body&&body.unparse(o)}
-         end"
+      def unparse o=default_unparse_options
+        result=unparse_nl(self,o)+"         for #{iterator.lhs_unparse(o)[1...-1]} in #{enumerable.unparse o}"
+        result+=unparse_nl(body,o)+"           #{body.unparse(o)}" if body
+        result+=";end"
       end
 
-      def parsetree
+      def parsetree(o)
 =begin
         case vars=@data[0]
         when Node: 
-          vars=vars.parsetree
+          vars=vars.parsetree(o)
           if vars.first==:call
             vars[0]=:attrasgn
             vars[2]="#{vars[2]}=".to_sym
@@ -3179,22 +3983,22 @@ class RedParse
         when Array: 
           vars=[:masgn, [:array, 
                   *vars.map{|lval| 
-                     res=lval.parsetree
-                     res[0]=lval.varname2assigntype if VarNameToken===lval
+                     res=lval.parsetree(o)
+                     res[0]=lval.varname2assigntype if VarNode===lval
                      res
                    } 
                ]]
-        when VarNameToken: 
+        when VarNode
           ident=vars.ident
-          vars=vars.parsetree
+          vars=vars.parsetree(o)
           (vars[0]=vars.varname2assigntype) rescue nil
         else fail
         end
 =end
 
-        vars=self.for.lvalue_parsetree
-        result=[:for, self.in.begin_parsetree, vars]
-        result.push self.do.parsetree if self.do
+        vars=self.for.lvalue_parsetree(o)
+        result=[:for, self.in.begin_parsetree(o), vars]
+        result.push self.do.parsetree(o) if self.do
         result
       end
 
@@ -3204,8 +4008,8 @@ class RedParse
     class HashLiteralNode<ValueNode
       def initialize(open,contents,close)
         case contents
-        when nil: super()
-        when ArrowOpNode: super(contents.first,contents.last)
+        when nil; super()
+        when ArrowOpNode; super(contents.first,contents.last)
         when CommaOpNode,Array
           if ArrowOpNode===contents.first
             data=[]
@@ -3214,21 +4018,25 @@ class RedParse
               data.push pair.first,pair.last
             }
           else
+            @no_arrows=true
             data=Array[*contents]
           end
           super(*data)
         end
-        @no_braces=true unless open
+        @no_braces=!open
       end
 
+      attr :no_arrows
+
       def image; "({})" end
 
-      def unparse o
+      def unparse o=default_unparse_options
         result=''
         result << "{" unless @no_braces
+        arrow= defined?(@no_arrows) ? " , " : " => "
         (0...size).step(2){|i| 
-          result<<
-            self[i].unparse(o)+' => '+
+          result<< unparse_nl(self[i],o,'')+
+            self[i].unparse(o)+arrow+
             self[i+1].unparse(o)+', '
         }
         result.chomp! ', '
@@ -3236,8 +4044,13 @@ class RedParse
         return result
       end
 
-      def parsetree
-        map{|elem| elem.rescue_parsetree}.unshift :hash
+      def parsetree(o)
+        map{|elem| elem.rescue_parsetree(o)}.unshift :hash
+      end
+
+      def error? rubyversion=1.8
+        return true if @no_arrows and rubyversion>=1.9
+        return super
       end
     end
 
@@ -3253,20 +4066,21 @@ class RedParse
 
       def image; "(?:)" end
 
-      def unparse o
+      def unparse o=default_unparse_options
         "#{condition.unparse o} ? #{consequent.unparse o} : #{otherwise.unparse o}"      
       end
 
       def elsifs; [] end
 
-      def parsetree
-        cond=condition.rescue_parsetree
+      def parsetree(o)
+        cond=condition.rescue_parsetree(o)
         cond[0]=:fcall if cond[0]==:vcall and cond[1].to_s[/[?!]$/]
-        [:if, cond, consequent.begin_parsetree, otherwise.begin_parsetree]
+        [:if, cond, consequent.begin_parsetree(o), otherwise.begin_parsetree(o)]
       end
     end
 
     class MethodNode<ValueNode
+      include HasRescue
       param_names(:defword_,:receiver,:name,:maybe_eq_,:args,:semi_,:body,:rescues,:elses,:ensures,:endword_)
       alias ensure_ ensures
       alias else_ elses
@@ -3276,7 +4090,7 @@ class RedParse
       
       def initialize(defword_,header,maybe_eq_,semi_,
                      body,rescues,else_,ensure_,endword_)
-        @empty_ensure=nil
+        @empty_else=@empty_ensure=nil
 #        if DotCallNode===header
 #          header=header.data[1]
 #        end
@@ -3302,13 +4116,14 @@ class RedParse
 
       attr_reader :empty_ensure, :empty_else
 
-      def receiver= x
-        self[0]=x      
-      end
 
-      def body= x
-        self[3]=x      
-      end
+#      def receiver= x
+#        self[0]=x      
+#      end
+#
+#      def body= x
+#        self[3]=x      
+#      end
 
       def ensure_= x
         self[5]=x      
@@ -3322,17 +4137,23 @@ class RedParse
         "(def #{receiver.image.+('.') if receiver}#{name})"
       end
 
-      def unparse o
-        result="
-         def #{receiver&&receiver.unparse(o)+'.'}#{name}#{
-           args&&'('+args.map{|arg| arg.unparse o}.join(',')+')'
-        }
-           #{body&&body.unparse(o)}
-        "
+      def unparse o=default_unparse_options
+        result=[
+         "def ",receiver&&receiver.unparse(o)+'.',name,
+           args ? '('+args.map{|arg| arg.unparse o}.join(',')+')' : unparse_nl(body||self,o)
+        ]
+        result<<unparse_and_rescues(o)
+=begin
+        body&&result+=body.unparse(o)
+
         result+=rescues.map{|resc| resc.unparse o}.to_s 
         result+="else #{else_.unparse o}\n"  if else_
+        result+="else\n" if @empty_else
         result+="ensure #{ensure_.unparse o}\n"  if ensure_
-        result+="end"
+        result+="ensure\n" if @empty_ensure
+=end
+        result<<unparse_nl(endline,o)+"end"
+        result.to_s
       end
 
       def to_lisp
@@ -3340,12 +4161,12 @@ class RedParse
         #no receiver, args, rescues, else_ or ensure_...
       end
 
-      def parsetree
+      def parsetree(o)
         name=name().to_sym
 
         result=[name, target=[:scope, [:block, ]] ]
         if receiver
-          result.unshift :defs, receiver.rescue_parsetree
+          result.unshift :defs, receiver.rescue_parsetree(o)
         else
           result.unshift :defn
         end
@@ -3365,21 +4186,21 @@ class RedParse
         end
 
         target.push args=[:args,]
-        target.push unamp.parsetree if unamp
+        target.push unamp.parsetree(o) if unamp
 
         if args()
           initvals=[]
           args().each{|arg| 
               case arg
-                  when VarNameToken:
+                  when VarNode
                     args.push arg.ident.to_sym
                   when UnaryStarNode:
                     args.push "*#{arg.val.ident}".to_sym
                   when UnOpNode:
                     nil
                   when AssignNode:
-                    initvals << arg.parsetree
-                    initvals[-1][-1]=arg.right.rescue_parsetree #ugly
+                    initvals << arg.parsetree(o)
+                    initvals[-1][-1]=arg.right.rescue_parsetree(o) #ugly
                     args.push arg[0].ident.to_sym
                   else 
                     fail "unsupported node type in method param list: #{arg}"
@@ -3388,18 +4209,22 @@ class RedParse
           unless initvals.empty?
             initvals.unshift(:block) 
             args << initvals
-            args[-2][0]==:block_arg and target.push args.delete_at(-2)
+            #args[-2][0]==:block_arg and target.push args.delete_at(-2)
           end
         end
         target.push [:nil] if !goodies && !receiver
+
+        #it would be better to use parsetree_and_rescues for the rest of this method,
+        #just to be DRYer
+
         target.push ensuretarget=target=[:ensure, ] if ensures or @empty_ensure
         #simple dup won't work... won't copy extend'd modules
         body=Marshal.load(Marshal.dump(body())) if body()
         elses=elses()
         if rescues.empty?
           case body
-          when SequenceNode: body << elses;elses=nil 
-          when nil: body=elses;elses=nil
+          when SequenceNode; body << elses;elses=nil
+          when nil; body=elses;elses=nil
           else nil
           end if elses
         else
@@ -3407,11 +4232,11 @@ class RedParse
           elses=elses()
         end
         if body
-          if ParenedNode===body and body.size>1 and 
+          if BeginNode===body||RescueOpNode===body and 
             body.rescues.empty? and !body.ensure and !body.empty_ensure and body.body and body.body.size>1
               wantblock=true
           end
-          body=body.parsetree
+          body=body.parsetree(o)
           if body.first==:block and rescues.empty? and not ensures||@empty_ensure
             if wantblock
               target.push body
@@ -3420,13 +4245,13 @@ class RedParse
               target.concat body
             end
           else
-            body=[:block, *body] if wantblock
+            #body=[:block, *body] if wantblock
             target.push body
           end
         end
-        target.push linked_list(rescues.map{|rescue_| rescue_.parsetree }) unless rescues.empty?
-        target.push elses.parsetree if elses
-        ensuretarget.push ensures.parsetree if ensures
+        target.push linked_list(rescues.map{|rescue_| rescue_.parsetree(o) }) unless rescues.empty?
+        target.push elses.parsetree(o) if elses
+        ensuretarget.push ensures.parsetree(o) if ensures
         ensuretarget.push [:nil] if @empty_ensure
 
         return result
@@ -3436,13 +4261,13 @@ class RedParse
     module BareSymbolUtils
       def baresym2str(node)
         case node
-        when MethNameToken:  node.ident
-        when VarNameToken: node
-        when LiteralNode: 
+        when MethNameToken;  node.ident
+        when VarNode; node
+        when LiteralNode
           case node.val
-          when Symbol: 
+          when Symbol 
             node.val.to_s
-          when StringNode: node.val
+          when StringNode; node.val
 #          when StringToken: StringNode.new(node.val)
           else fail
           end
@@ -3451,20 +4276,21 @@ class RedParse
 
       def str_unparse(str,o)
        case str
-       when VarNameToken: str.ident
-       when String: 
+       when VarNode; str.ident
+       when "~@"; str
+       when String 
          str.to_sym.inspect
          #what if str isn't a valid method or operator name? should be quoted
-       when StringNode: 
+       when StringNode
          ":"+str.unparse(o)
        else fail
        end
       end
 
-      def str2parsetree(str)
+      def str2parsetree(str,o)
        if String===str then [:lit, str.to_sym] 
        else 
-         result=str.parsetree
+         result=str.parsetree(o)
          result[0]=:dsym
          result
        end
@@ -3480,16 +4306,16 @@ class RedParse
         super(to,from)
       end
 
-      def unparse o
+      def unparse o=default_unparse_options
         "alias #{str_unparse to,o} #{str_unparse from,o}"      
       end
 
       def image; "(alias)" end
-      def parsetree
-        if VarNameToken===to and to.ident[0]==?$
+      def parsetree(o)
+        if VarNode===to and to.ident[0]==?$
           [:valias, to.ident.to_sym, from.ident.to_sym]
         else
-          [:alias, str2parsetree(to), str2parsetree(from)]
+          [:alias, str2parsetree(to,o), str2parsetree(from,o)]
         end
       end
     end
@@ -3507,12 +4333,12 @@ class RedParse
 
       def image; "(undef)" end
 
-      def unparse o
+      def unparse o=default_unparse_options
         "undef #{map{|name| str_unparse name,o}.join(', ')}"
       end
 
-      def parsetree
-        result=map{|name| [:undef, str2parsetree(name)] }
+      def parsetree(o)
+        result=map{|name| [:undef, str2parsetree(name,o)] }
         if result.size==1
           result.first
         else
@@ -3522,15 +4348,29 @@ class RedParse
     end
 
     class NamespaceNode<ValueNode
+      include HasRescue
+      def initialize(*args)
+        @empty_ensure=@empty_else=nil
+        super
+      end
     end
  
     class ModuleNode<NamespaceNode
-      param_names(:moduleword_,:name,:semi_,:body,:endword_)
+      param_names(:name,:body,:rescues,:else!,:ensure!)
+
+      def initialize moduleword,name,semiword,body,rescues,else_,ensure_,endword
+        else_=else_.val if else_
+        ensure_=ensure_.val if ensure_
+        super(name,body,rescues,else_,ensure_)        
+      end
+
+      alias else_ else
+      alias ensure_ ensure
 
       def image; "(module #{name})" end
 
-      def unparse o
-        "module #{name.unparse o}\n#{body&&body.unparse(o)}\nend"
+      def unparse o=default_unparse_options
+        "module #{name.unparse o}#{unparse_nl(body||self,o)}#{unparse_and_rescues(o)};end"
       end
 
       def parent; nil end
@@ -3543,38 +4383,43 @@ class RedParse
         return result
       end
 
-      def parsetree
+      def parsetree(o)
         name=name()
-        if VarNameToken===name
+        if VarNode===name
           name=name.ident.to_sym
         elsif name.nil? #do nothing
-#        elsif quirks
+#        elsif o[:quirks]
 #          name=name.constant.ident.to_sym
         else 
-          name=name.parsetree
+          name=name.parsetree(o)
         end
         result=[:module, name, scope=[:scope, ]]
-        body&&scope << body.parsetree
+        scope << parsetree_and_rescues(o) if body
         return result
       end
     end
 
     class ClassNode<NamespaceNode
-      param_names(:name,:parent,:body)
-      def initialize(classword,name,semi,body,endword)
+      param_names(:name,:parent,:body,:rescues,:else!,:ensure!)
+      def initialize(classword,name,semi,body,rescues, else_, ensure_, endword)
         if OpNode===name
           name,op,parent=*name
           op=="<" or fail "invalid superclass expression: #{name}"
         end
-        super(name,parent,body)
+        else_=else_.val if else_
+        ensure_=ensure_.val if ensure_
+        super(name,parent,body,rescues,else_,ensure_)
       end
 
+      alias else_ else
+      alias ensure_ ensure
+
       def image; "(class #{name})" end
 
-      def unparse o
+      def unparse o=default_unparse_options
         result="class #{name.unparse o}"
         result+=" < #{parent.unparse o}" if parent
-        result+="\n#{body&&body.unparse(o)}\nend"
+        result+=unparse_nl(body||self,o)+"#{unparse_and_rescues(o)};end"
         return result
       end
 
@@ -3586,39 +4431,48 @@ class RedParse
         return result
       end
 
-      def parsetree
+      def parsetree(o)
         name=name()
-        if VarNameToken===name
+        if VarNode===name
           name=name.ident.to_sym
         elsif name.nil? #do nothing
-#        elsif quirks
+#        elsif o[:quirks]
 #          name=name.constant.ident.to_sym
         else 
-          name=name.parsetree
+          name=name.parsetree(o)
         end
-        result=[:class, name, parent&&parent.parsetree, scope=[:scope,]]
-        body&&scope << body.parsetree
+        result=[:class, name, parent&&parent.parsetree(o), scope=[:scope,]]
+        scope << parsetree_and_rescues(o) if body
         return result
       end
     end
 
     class MetaClassNode<NamespaceNode
-      param_names :classword_, :leftleft_, :val, :semi_, :body, :endword_
+      param_names :val, :body, :rescues,:else!,:ensure!
+      def initialize classword, leftleftword, val, semiword, body, rescues,else_,ensure_, endword
+        else_=else_.val if else_
+        ensure_=ensure_.val if ensure_
+        super(val,body,rescues,else_,ensure_)
+      end 
+
       alias expr val
       alias object val
       alias obj val
       alias receiver val
       alias name val
 
+      alias ensure_ ensure
+      alias else_ else
+
       def image; "(class<<)" end
 
-      def unparse o
-        "class << #{obj.unparse o}\n#{body&&body.unparse(o)}\nend"
+      def unparse o=default_unparse_options
+        "class << #{obj.unparse o}#{unparse_nl(body||self,o)}#{unparse_and_rescues(o)};end"
       end
 
-      def parsetree
-        result=[:sclass, expr.parsetree, scope=[:scope]]
-        body&&scope << body.parsetree
+      def parsetree(o)
+        result=[:sclass, expr.parsetree(o), scope=[:scope]]
+        scope << parsetree_and_rescues(o) if body
         return result
       end
     end
@@ -3628,7 +4482,7 @@ class RedParse
       def initialize(rescueword,arrowword,exceptions,thenword)
         case exceptions
         when nil
-        when VarNameToken:
+        when VarNode:
           if arrowword
             exvarname=exceptions
             exceptions=nil
@@ -3646,7 +4500,7 @@ class RedParse
           exceptions=Array.new(exceptions)
         end
         fail if arrowword
-#        fail unless VarNameToken===exvarname || exvarname.nil?
+#        fail unless VarNode===exvarname || exvarname.nil?
         super(exceptions,exvarname)
       end
 
@@ -3662,17 +4516,18 @@ class RedParse
         super(exlist,rescuehdr.varname,action)
       end
 
-      def unparse o
+      def unparse o=default_unparse_options
         xx=exceptions.map{|exc| exc.unparse o}.join(',')
-        "rescue #{xx} #{varname&&'=> '+varname.lhs_unparse(o)}\n#{action&&action.unparse(o)}\n"
+        unparse_nl(self,o)+
+        "rescue #{xx} #{varname&&'=> '+varname.lhs_unparse(o)}#{unparse_nl(action||self,o)}#{action&&action.unparse(o)}"
       end
 
-      def parsetree
+      def parsetree(o)
         result=[:resbody, nil]
         fail unless exceptions.class==Array
-        ex=#if Node===exceptions; [exceptions.rescue_parsetree] 
+        ex=#if Node===exceptions; [exceptions.rescue_parsetree(o)] 
            #elsif exceptions
-             exceptions.map{|exc| exc.rescue_parsetree} 
+             exceptions.map{|exc| exc.rescue_parsetree(o)} 
            #end
         if !ex or ex.empty? #do nothing
         elsif ex.last.first!=:splat
@@ -3682,19 +4537,19 @@ class RedParse
         else
           result[1]= [:argscat, ex[0...-1].unshift(:array), ex.last[1]]
         end
-        VarNameToken===varname and offset=varname.offset
+        VarNode===varname and offset=varname.offset
         action=if varname
           SequenceNode.new(
             AssignNode.new( 
               varname, 
               KeywordToken.new("=",offset),
-              VarNameToken.new("$!",offset)
+              VarNode.new(VarNameToken.new("$!",offset))
             ),nil,action()
           )
         else
           action()
         end
-        result.push action.parsetree if action
+        result.push action.parsetree(o) if action
         result
       end
 
@@ -3705,8 +4560,8 @@ class RedParse
       param_names(:receiver,:lbrack_,:params,:rbrack_)
       def initialize(receiver,lbrack,params,rbrack)
         params=case params
-        when CommaOpNode: Array.new params
-        when nil:
+        when CommaOpNode; Array.new params
+        when nil
         else [params]
         end
         super(receiver,params)
@@ -3714,7 +4569,7 @@ class RedParse
 
       def image; "(#{receiver.image}.[])" end
 
-      def unparse o
+      def unparse o=default_unparse_options
         [ receiver.unparse(o).sub(/\s+\Z/,''),
           '[',
           params&&params.map{|param| param.unparse o}.join(','),
@@ -3722,18 +4577,18 @@ class RedParse
         ].to_s
       end
 
-      def parsetree
-        result=parsetree_no_fcall
-        quirks and VarLikeNode===receiver and receiver.name=='self' and
+      def parsetree(o)
+        result=parsetree_no_fcall o
+        o[:quirks] and VarLikeNode===receiver and receiver.name=='self' and
           result[0..2]=[:fcall,:[]] 
         return result
       end
 
-      def parsetree_no_fcall
+      def parsetree_no_fcall o
         params=params()
-        output,star,amp=param_list_parse(params)
-#        receiver=receiver.parsetree
-        result=[:call, receiver.parsetree, :[], output]
+        output,star,amp=param_list_parse(params,o)
+#        receiver=receiver.parsetree(o)
+        result=[:call, receiver.rescue_parsetree(o), :[], output]
         if params
           if star and params.size==1
             output.concat star
@@ -3746,27 +4601,47 @@ class RedParse
         end
         return result
       end
-      def lvalue_parsetree
-        result=parsetree_no_fcall
+      def lvalue_parsetree(o)
+        result=parsetree_no_fcall o
         result[0]=:attrasgn
         result[2]=:[]=
         result
       end
 
-      def lvalue?
+      def lvalue
         return @lvalue if defined? @lvalue
         @lvalue=true
       end
-      attr_accessor :lvalue
+      attr_writer :lvalue
+      identity_param :lvalue, nil, true
     end
 
 
-    class StartNode<Node  #not to appear in final tree
+    class StartToken<Token  #not to appear in final tree
       def initialize; end
 
       def image; "(START)" end
+      alias to_s image
+
+
     end #beginning of input marker
 
+    class EoiToken
+      #hack hack: normally, this should never
+      #be called, but it may be when input is empty now.
+      def to_parsetree(*options)
+        []
+      end
+    end
+
+    class GoalPostToken<Token  #not to appear in final tree
+      def initialize(offset); @offset=offset end
+      def ident; "|" end
+      attr :offset
+
+      def image; "|" end
+    end
+
     class GoalPostNode<Node  #not to appear in final tree
       def initialize(offset); @offset=offset end
       def ident; "|" end
@@ -3776,16 +4651,8 @@ class RedParse
     end
 
     module ErrorNode
-      #pass the buck to child ErrorNodes until there's no one else
-      def blame
-        middle.each{|node| 
-          node.respond_to? :blame and return node.blame 
-        }
-        return self
-      end
- 
-
-      #def msg; ... end
+      def error?(x=nil) @error end
+      alias msg error?
     end
 
 
@@ -3797,9 +4664,19 @@ class RedParse
 
       def image; "misparsed #{what}" end
 
-      def msg
-        "#@line: misparsed #{what}: #{middle.map{|node| node&&node.image}.join(' ')}" 
+      #pass the buck to child ErrorNodes until there's no one else
+      def blame
+        middle.each{|node| 
+          node.respond_to? :blame and return node.blame 
+        }
+        return self
+      end
+ 
+      def error? x=nil
+        inner=middle.grep(MisparsedNode).first and return inner.error?( x )
+        "#@endline: misparsed #{what}: #{middle.map{|node| node&&node.image}.join(' ')}" 
       end
+      alias msg error?
     end
 
 #  end