jruby-parser 0.3

data/bin/generate_parser

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Run from your JRuby home directory....More smarts needed here.
+
+###### Change these to tastes ######
+JAY=jay
+RUBY=ruby
+PARSER_BASE=DefaultRubyParser
+YYTABLE_PREFIX=
+DEBUG=true
+###### Do not change below ######
+
+if [ "$1" != "" ]; then
+  PARSER_BASE=$1
+fi
+shift
+
+if [ "$1" != "" ]; then
+  YYTABLE_PREFIX=$1
+fi
+
+if [ "$DEBUG" != "" ]; then
+  DEBUG_FLAG=-t
+  # Nonesense...my script-fu is weak
+  DEBUG_STRIP="xdyhbk"
+else
+  DEBUG_FLAG=
+  DEBUG_STRIP="^//t"
+fi
+
+echo "Generating Parser '$PARSER_BASE' w/ YYTable prefix of '$YYTABLE_PREFIX'"
+
+PARSER_DIR=src/org/jrubyparser/parser
+
+pushd $PARSER_DIR
+
+# Generate grammar as intermediate file
+$JAY $DEBUG_FLAG $PARSER_BASE.y < skeleton.parser | grep -v $DEBUG_STRIP >$PARSER_BASE.out
+
+# Patch file to get around Java static initialization issues plus extract
+# a bunch of stuff to seperate file (yytables).
+$RUBY ../../../../bin/patch_parser.rb $PARSER_BASE.out $YYTABLE_PREFIX > $PARSER_BASE.out2
+$RUBY ../../../../bin/optimize_parser.rb $PARSER_BASE.out2 $YYTABLE_PREFIX > $PARSER_BASE.java
+rm -f $PARSER_BASE.out $PARSER_BASE.out2
+
+popd

data/bin/optimize_parser.rb

@@ -0,0 +1,106 @@
+class PostProcessor
+  def initialize(source, out=STDOUT)
+    @out = out
+    @lines = File.readlines(source)
+    @index = -1
+    @case_bodies = {}
+    @max_case_number = -1
+  end
+
+  # Read/Unread with ability to push back one line for a single lookahead
+  def read
+    @index += 1
+    line = @last ? @last : @lines[@index]
+    @last = nil
+    line
+  end
+
+  def unread(line)
+    @index -= 1
+    @last = line
+  end
+
+  def end_of_actions?(line)
+     return line =~ %r{^//\s*ACTIONS_END}
+  end
+
+  def translate
+    while (line = read)
+      if line =~ %r{^//\s*ACTIONS_BEGIN}
+        translate_actions
+      elsif line =~ %r{^//\s*ACTION_BODIES}
+        generate_action_body_methods
+      else
+        @out.puts line
+      end
+    end
+  end
+
+  def generate_action_body_methods
+    @out.puts "static ParserState[] states = new ParserState[#{@max_case_number+1}];"
+    @out.puts "static {";
+    @case_bodies.each do |state, code_body| 
+      generate_action_body_method(state, code_body) 
+    end
+    @out.puts "}";
+  end
+
+  def generate_action_body_method(state, code_body)
+    @out.puts "states[#{state}] = new ParserState() {"
+    @out.puts "  public Object execute(ParserSupport support, Lexer lexer, Object yyVal, Object[] yyVals, int yyTop) {"
+    code_body.each { |line| @out.puts line }
+    @out.puts "    return yyVal;"
+    @out.puts "  }"
+    @out.puts "};"
+  end
+
+  def translate_actions
+    count = 1
+    while (translate_action)
+      count += 1
+    end
+  end
+
+  # Assumptions:
+  # 1. no break; in our code.  A bit weak, but this is highly specialized code.
+  # 2. All productions will have a line containing only { (with optional comment)
+  # 3. All productions will end with a line containly only } followed by break in ass 1.
+  def translate_action
+    line = read
+    return false if end_of_actions?(line) || line !~ /case\s+(\d+):/
+    case_number = $1.to_i
+
+    line = read
+    return false if line !~ /line\s+(\d+)/
+    line_number = $1
+
+    # Extra boiler plate '{' that we do not need
+    line = read
+    return false if line !~ /^\s*\{\s*(\/\*.*\*\/)?$/
+
+    @max_case_number = case_number if case_number > @max_case_number
+
+    label = "case#{case_number}_line#{line_number}"
+
+    body = []
+    last_line = nil
+    while (line = read)
+      if line =~ /^\s*\}\s*$/ # Extra trailing boiler plate
+        next_line = read
+        if next_line =~ /break;/
+          break
+        else
+          body << line
+          unread next_line
+        end
+      else
+        body << line
+      end
+    end
+
+    @case_bodies[case_number] = body
+    true
+  end
+end
+
+PostProcessor.new(ARGV.shift).translate

data/bin/patch_parser.rb

@@ -0,0 +1,105 @@
+def get_numbers_until_end_block(table)
+  while gets
+    break if /\};/ =~ $_
+    next if /^\/\// =~ $_
+    split(/,/).each do |number|
+      n = number.strip
+      table.push(n.to_i) unless n == ""
+    end
+  end
+  table
+end
+
+while gets
+  break if /protected static final short\[\] yyTable = \{/ =~ $_
+  print $_
+end
+
+# A little hacky...gets before ARGV to shift off and open file
+yytable_prefix = ARGV.shift || ''
+
+table4 = get_numbers_until_end_block([])
+
+puts "    protected static final short[] yyTable = #{yytable_prefix}YyTables.yyTable();"
+
+while gets
+  break if /protected static final short\[\] yyCheck = \{/ =~ $_
+  print $_
+end
+
+check4 = get_numbers_until_end_block([])
+
+puts "    protected static final short[] yyCheck = #{yytable_prefix}YyTables.yyCheck();"
+
+while gets
+  print $_
+end
+
+table2 = table4.slice!(0, table4.size / 2)
+table3 = table4.slice!(0, table4.size / 2)
+table1 = table2.slice!(0, table2.size / 2)
+check2 = check4.slice!(0, check4.size / 2)
+check3 = check4.slice!(0, check4.size / 2)
+check1 = check2.slice!(0, check2.size / 2)
+
+def printShortArray(table, f)
+  table.each_with_index { |e, i|
+    f.print "\n         " if (i % 10 == 0)
+    begin
+      f.printf "%4d, ", e
+    rescue ArgumentError => a
+      $stderr.puts "Trouble printing '#{e}' on index #{i}"
+    end
+  }
+end
+
+def printShortMethod(f, table, name)
+  f.puts "   private static final short[] yy#{name}() {"
+  f.puts "      return new short[] {"
+  printShortArray table, f
+  f.puts
+  f.puts "      };"
+  f.puts "   }"
+  f.puts
+end
+
+open("#{yytable_prefix}YyTables.java", "w") { |f|
+  f.print <<END
+package org.jrubyparser.parser;
+
+public class #{yytable_prefix}YyTables {
+   private static short[] combine(short[] t1, short[] t2, 
+                                  short[] t3, short[] t4) {
+      short[] t = new short[t1.length + t2.length + t3.length + t4.length];
+      int index = 0;
+      System.arraycopy(t1, 0, t, index, t1.length);
+      index += t1.length;
+      System.arraycopy(t2, 0, t, index, t2.length);
+      index += t2.length;
+      System.arraycopy(t3, 0, t, index, t3.length);
+      index += t3.length;
+      System.arraycopy(t4, 0, t, index, t4.length);
+      return t;
+   }
+
+   public static final short[] yyTable() {
+      return combine(yyTable1(), yyTable2(), yyTable3(), yyTable4());
+   }
+
+   public static final short[] yyCheck() {
+      return combine(yyCheck1(), yyCheck2(), yyCheck3(), yyCheck4());
+   }
+END
+
+  printShortMethod(f, table1, "Table1")
+  printShortMethod(f, table2, "Table2")
+  printShortMethod(f, table3, "Table3")
+  printShortMethod(f, table4, "Table4")
+
+  printShortMethod(f, check1, "Check1")
+  printShortMethod(f, check2, "Check2")
+  printShortMethod(f, check3, "Check3")
+  printShortMethod(f, check4, "Check4")
+
+  f.puts "}"
+}

data/lib/jruby-parser/core_ext/array.rb

@@ -0,0 +1,10 @@
+require 'jruby'
+
+class Array
+  def to_ast_node(position = nil)
+    inject(org.jrubyparser.ast.ArrayNode.new(position)) do |array, value|
+      value = value.to_ast_node if value.respond_to? :to_ast_node
+      array << value
+    end
+  end
+end

data/lib/jruby-parser/core_ext/attr_assign_node.rb

@@ -0,0 +1,6 @@
+require 'java'
+require 'jruby-parser/util/coercer'
+
+class org::jrubyparser::ast::AttrAssignNode
+  include JRubyParser::Receiver, JRubyParser::Args
+end

data/lib/jruby-parser/core_ext/boolean.rb

@@ -0,0 +1,11 @@
+class TrueClass
+  def to_ast_node(position=nil)
+    org.jrubyparser.ast.TrueNode.new(position)
+  end
+end
+
+class FalseClass
+  def to_ast_node(position=nil)
+    org.jrubyparser.ast.FalseNode.new(position)
+  end
+end

data/lib/jruby-parser/core_ext/call_node.rb

@@ -0,0 +1,6 @@
+require 'java'
+require 'jruby-parser/util/coercer'
+
+class org::jrubyparser::ast::CallNode
+  include JRubyParser::Receiver, JRubyParser::Value, JRubyParser::Args
+end

data/lib/jruby-parser/core_ext/fcall_node.rb

@@ -0,0 +1,6 @@
+require 'java'
+require 'jruby-parser/util/coercer'
+
+class org::jrubyparser::ast::FCallNode
+  include JRubyParser::Args
+end

data/lib/jruby-parser/core_ext/list_node.rb

@@ -0,0 +1,10 @@
+require 'java'
+require 'jruby-parser/util/coercer'
+
+class org::jrubyparser::ast::ListNode
+  def <<(value)
+    value = value.to_ast_node if value.respond_to? :to_ast_node
+    value.position = getPosition unless value.position
+    add(value)
+  end
+end

data/lib/jruby-parser/core_ext/nil.rb

@@ -0,0 +1,7 @@
+require 'java'
+
+class NilClass
+  def to_ast_node(position=nil)
+    org.jrubyparser.ast.NilNode.new(position)
+  end
+end

data/lib/jruby-parser/core_ext/node.rb

@@ -0,0 +1,66 @@
+require 'java'
+
+class org::jrubyparser::ast::Node
+  include Enumerable
+  
+  ##
+  # Find nth child element of this node
+  #
+  def [](value)
+    child_nodes[value]
+  end
+
+  ##
+  # Replace the nth child element of this node with the specified value.
+  # if the value is an actual Ruby value it will attempt to call to_ast_node
+  # on it to do automatic coercion to an AST node.  If the node does not
+  # contain positioning information then it will just use the
+  # old nodes information
+  def []=(index, value)
+    value = value.to_ast_node if value.respond_to? :to_ast_node
+
+    old_value = child_nodes[index]
+    value.position = old_value.position unless value.position
+    child_nodes[index] = value
+  end
+
+  ##
+  # Find first node by name (which is short_name of actual node)
+  # === parameters
+  # * _name_ is the name of the class you want to find
+  # === examples
+  # 
+  # root.find(:fcall) # Find first child node of type fcall (depth-first)
+  #
+  def find_type(name, &block)
+    name = name.to_s
+    return self if name == short_name && (!block_given? || yield(self))
+
+    child_nodes.each do |child|
+      value = child.find_type(name, &block)
+
+      return value if value
+    end
+    nil
+  end
+  alias find_node find_type
+  
+  def each(&block)
+    yield self
+    child_nodes.each { |child| child.each(&block) }
+  end
+  
+  ##
+  # Convert this node back to human-readable source code.
+  #
+  def to_source(opts = {})
+    filename = opts[:filename] ? opts[:filename] : '(string)'
+    java.io.StringWriter.new.tap do |writer|
+      accept org.jrubyparser.rewriter.ReWriteVisitor.new(writer, filename)
+    end.to_s
+  end
+
+  def short_name
+    java_class.name.gsub(/(^.*\.|Node$)/, '').downcase
+  end
+end

data/lib/jruby-parser/core_ext/numeric.rb

@@ -0,0 +1,13 @@
+require 'java'
+
+class Fixnum
+  def to_ast_node(position=nil)
+    org.jrubyparser.ast.FixnumNode.new(position, self)
+  end
+end
+
+class Float
+  def to_ast_node(position=nil)
+    org.jrubyparser.ast.FloatNode.new(position, self)
+  end
+end

data/lib/jruby-parser/core_ext/op_element_asgn_node.rb

@@ -0,0 +1,6 @@
+require 'java'
+require 'jruby-parser/util/coercer'
+
+class org::jrubyparser::ast::OpElementAsgnNode
+  include JRubyParser::Receiver, JRubyParser::Value, JRubyParser::Args
+end

data/lib/jruby-parser/core_ext/string.rb

@@ -0,0 +1,7 @@
+require 'java'
+
+class String
+  def to_ast_node(position=nil)
+    org.jrubyparser.ast.StrNode.new(position, self.to_java(:string))
+  end
+end

data/lib/jruby-parser/core_ext/super_node.rb

@@ -0,0 +1,6 @@
+require 'java'
+require 'jruby-parser/util/coercer'
+
+class org::jrubyparser::ast::SuperNode
+  include JRubyParser::Args
+end

data/lib/jruby-parser/util/coercer.rb

@@ -0,0 +1,44 @@
+module JRubyParser
+  module Receiver
+    def self.included(cls)
+      cls.class_eval do
+        def receiver=(value)
+           value = value.to_ast_node if value.respond_to? :to_ast_node
+           old_value = getReceiver
+           value.position = old_value.position unless value.position    
+           setReceiver(value)
+        end
+      end
+    end
+  end
+  
+  module Value
+    def self.included(cls)
+      cls.class_eval do
+        def value=(value)
+           value = value.to_ast_node if value.respond_to? :to_ast_node
+           old_value = getValue
+           value.position = old_value.position unless value.position    
+           setValue(value)
+        end
+      end
+    end
+  end  
+  
+  module Args
+    def self.included(cls)
+      cls.class_eval do
+        def args=(value)
+           value = value.to_ast_node if value.respond_to? :to_ast_node
+           old_value = getArgs
+           unless value.position
+             value.position = old_value.position 
+             value.each { |e| e.position = old_value.position } #if value.respond_to? :each
+           end
+           
+           setArgs(value)
+        end
+      end
+    end
+  end
+end

data/lib/jruby-parser/version.rb

@@ -0,0 +1,3 @@
+module JRubyParser
+  VERSION = "0.3"
+end

data/lib/jruby-parser.rb

@@ -0,0 +1,36 @@
+require 'java'
+require 'jruby-parser.jar'
+require 'jruby-parser/core_ext/array'
+require 'jruby-parser/core_ext/boolean'
+require 'jruby-parser/core_ext/nil'
+require 'jruby-parser/core_ext/call_node'
+require 'jruby-parser/core_ext/fcall_node'
+require 'jruby-parser/core_ext/list_node'
+require 'jruby-parser/core_ext/node'
+require 'jruby-parser/core_ext/numeric' # float,fixnum
+require 'jruby-parser/core_ext/op_element_asgn_node'
+require 'jruby-parser/core_ext/string'
+
+module JRubyParser
+  Compat = org.jrubyparser.CompatVersion
+
+  ##
+  # Parse source string and return a Abstract Syntax Tree (AST) of the source.
+  # You may also pass in additional options to affect the reported filename
+  # and which version of Ruby you want to use:
+  # 
+  # === Parameters
+  # * _source_string_ source you want to parse
+  # * _opts_ customize how your source is parsed (:filename, and :version [defaults to 1.9])
+  # === Example
+  # JRubyParser.parse(%q{puts "hello world"}, :version => JRubyParser::Compat::RUBY1_8)
+  # 
+  def parse(source_string, opts={})
+    filename = opts[:filename] ?  opts[:filename] : '(string)'
+    version = opts[:version] ?  opts[:version] : Compat::RUBY1_9
+    config = org.jrubyparser.parser.ParserConfiguration.new(0, version)
+    reader = java.io.StringReader.new(source_string)
+    org.jrubyparser.Parser.new.parse(filename, reader, config)
+  end
+  module_function :parse
+end

data/sample/simple.rb

@@ -0,0 +1,20 @@
+require 'jruby-parser'
+
+# Extra options hash can be passed in to override parser configuration
+# opts = {:version => JRubyParser::Compat::RUBY1_8, :filename => 'name.rb'}
+# root = JRubyParser.parse("b = foo(1)", opts)
+root = JRubyParser.parse("b = foo(1)")
+
+# Enumerable is mixed into AST tree
+# fcall = foot.find { |e| e.short_name == "fcall" }
+# ...but find_node is pretty common for spec writing:
+fcall = root.find_node(:fcall)
+
+# Change the AST.
+fcall.name = 'bar'
+
+# Notice this should be a TrueNode, but true knows to coerce into TrueNode
+fcall.args[0] = true
+
+# Write out the new source "b = bar(1)"
+p root.to_source

data/spec/ast/node_path.rb

@@ -0,0 +1,20 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + "/../helpers"
+$LOAD_PATH.unshift File.dirname(__FILE__) + "/../../lib"
+require 'jruby-parser'
+require 'parser_helpers'
+require 'node_helpers'
+
+describe Parser do
+  it "should parse alias with quotationmarks" do
+    root = parse <<-EOF
+def foo(arg)
+  puts arg
+end
+    EOF
+    list = root.pathTo(root.find_node(:defn))
+    list.size.should == 3
+    list.root.node_name.should == "RootNode"
+    list.leaf.node_name.should == "DefnNode"
+  end
+end
+

data/spec/helpers/node_helpers.rb

@@ -0,0 +1,120 @@
+require 'java'
+
+import org.jrubyparser.SourcePosition
+import org.jrubyparser.ast.Node
+
+class SourcePosition
+  def to_a
+    [start_line, end_line, start_offset, end_offset]
+  end
+end
+
+###########
+
+class AstPositionMatcher
+  def initialize(*args)
+    @position = args
+  end
+ 
+  def matches?(actual)
+    @actual = actual
+    actual.position.to_a == @position
+  end
+ 
+  def failure_message
+    return %[expected #{@actual.position.to_a.inspect} to have position #{@position.inspect}]
+  end
+ 
+  def negative_failure_message
+    return %[expected #{@actual.position.to_a.inspect} to not have position #{@position.inspect}]
+  end
+end
+ 
+module HavePosition
+  def have_position(*args)
+    AstPositionMatcher.new(*args)
+  end
+end
+
+class AstNameMatcher
+  def initialize(name)
+    @name = name
+  end
+ 
+  def matches?(actual)
+    @actual = actual
+    actual.name == @name
+  end
+ 
+  def failure_message
+    return %[expected #{@actual.inspect} to have name #{@name.inspect}]
+  end
+ 
+  def negative_failure_message
+    return %[expected #{@actual.inspect} to not have name #{@name.inspect}]
+  end
+end
+ 
+module HaveName
+  def have_name(name)
+    AstNameMatcher.new(name)
+  end
+end
+
+class AstNameAndPositionMatcher
+  def initialize(name, *args)
+    @name, @position = name, args
+  end
+ 
+  def matches?(actual)
+    @actual = actual
+    actual.name == @name && actual.position.to_a == @position
+  end
+ 
+  def failure_message
+    return %[expected #{@actual.name.inspect}, #{@actual.position.to_a.inspect} to have name and position #{@name.inspect}, #{@position.inspect}]
+  end
+ 
+  def negative_failure_message
+    return %[expected #{@actual.name.inspect}, #{@actual.position.to_a.inspect} to not have and position #{@name.inspect}, #{@position.inspect}]
+  end
+end
+
+module HaveNameAndPosition
+  def have_name_and_position(*args)
+    AstNameAndPositionMatcher.new(*args)
+  end
+end
+
+class ArgCountsMatcher
+  def initialize(pre, optional, rest, block)
+    @args = [pre, optional, rest, block]
+  end
+
+  def matches?(args)
+    @actual = [args.pre_count, args.optional_count, args.rest != nil, args.block != nil]
+    @args == @actual
+  end
+
+  def failure_message
+    return %[expected #{@actual.inspect} to have name #{@args.inspect}]
+  end
+
+  def negative_failure_message
+    return %[expected #{@actual.inspect} to not have name #{@args.inspect}]
+  end
+end
+
+module HaveArgCounts
+  def have_arg_counts(*args)
+    ArgCountsMatcher.new(*args)
+  end
+end
+ 
+#module Spec::Example::ExampleMethods
+class Object
+  include HavePosition
+  include HaveName
+  include HaveNameAndPosition
+  include HaveArgCounts
+end