jruby-parser 0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,46 @@
1
+ #!/bin/sh
2
+
3
+ # Run from your JRuby home directory....More smarts needed here.
4
+
5
+ ###### Change these to tastes ######
6
+ JAY=jay
7
+ RUBY=ruby
8
+ PARSER_BASE=DefaultRubyParser
9
+ YYTABLE_PREFIX=
10
+ DEBUG=true
11
+ ###### Do not change below ######
12
+
13
+ if [ "$1" != "" ]; then
14
+ PARSER_BASE=$1
15
+ fi
16
+ shift
17
+
18
+ if [ "$1" != "" ]; then
19
+ YYTABLE_PREFIX=$1
20
+ fi
21
+
22
+ if [ "$DEBUG" != "" ]; then
23
+ DEBUG_FLAG=-t
24
+ # Nonesense...my script-fu is weak
25
+ DEBUG_STRIP="xdyhbk"
26
+ else
27
+ DEBUG_FLAG=
28
+ DEBUG_STRIP="^//t"
29
+ fi
30
+
31
+ echo "Generating Parser '$PARSER_BASE' w/ YYTable prefix of '$YYTABLE_PREFIX'"
32
+
33
+ PARSER_DIR=src/org/jrubyparser/parser
34
+
35
+ pushd $PARSER_DIR
36
+
37
+ # Generate grammar as intermediate file
38
+ $JAY $DEBUG_FLAG $PARSER_BASE.y < skeleton.parser | grep -v $DEBUG_STRIP >$PARSER_BASE.out
39
+
40
+ # Patch file to get around Java static initialization issues plus extract
41
+ # a bunch of stuff to seperate file (yytables).
42
+ $RUBY ../../../../bin/patch_parser.rb $PARSER_BASE.out $YYTABLE_PREFIX > $PARSER_BASE.out2
43
+ $RUBY ../../../../bin/optimize_parser.rb $PARSER_BASE.out2 $YYTABLE_PREFIX > $PARSER_BASE.java
44
+ rm -f $PARSER_BASE.out $PARSER_BASE.out2
45
+
46
+ popd
@@ -0,0 +1,106 @@
1
+ class PostProcessor
2
+ def initialize(source, out=STDOUT)
3
+ @out = out
4
+ @lines = File.readlines(source)
5
+ @index = -1
6
+ @case_bodies = {}
7
+ @max_case_number = -1
8
+ end
9
+
10
+ # Read/Unread with ability to push back one line for a single lookahead
11
+ def read
12
+ @index += 1
13
+ line = @last ? @last : @lines[@index]
14
+ @last = nil
15
+ line
16
+ end
17
+
18
+ def unread(line)
19
+ @index -= 1
20
+ @last = line
21
+ end
22
+
23
+ def end_of_actions?(line)
24
+ return line =~ %r{^//\s*ACTIONS_END}
25
+ end
26
+
27
+ def translate
28
+ while (line = read)
29
+ if line =~ %r{^//\s*ACTIONS_BEGIN}
30
+ translate_actions
31
+ elsif line =~ %r{^//\s*ACTION_BODIES}
32
+ generate_action_body_methods
33
+ else
34
+ @out.puts line
35
+ end
36
+ end
37
+ end
38
+
39
+ def generate_action_body_methods
40
+ @out.puts "static ParserState[] states = new ParserState[#{@max_case_number+1}];"
41
+ @out.puts "static {";
42
+ @case_bodies.each do |state, code_body|
43
+ generate_action_body_method(state, code_body)
44
+ end
45
+ @out.puts "}";
46
+ end
47
+
48
+ def generate_action_body_method(state, code_body)
49
+ @out.puts "states[#{state}] = new ParserState() {"
50
+ @out.puts " public Object execute(ParserSupport support, Lexer lexer, Object yyVal, Object[] yyVals, int yyTop) {"
51
+ code_body.each { |line| @out.puts line }
52
+ @out.puts " return yyVal;"
53
+ @out.puts " }"
54
+ @out.puts "};"
55
+ end
56
+
57
+ def translate_actions
58
+ count = 1
59
+ while (translate_action)
60
+ count += 1
61
+ end
62
+ end
63
+
64
+ # Assumptions:
65
+ # 1. no break; in our code. A bit weak, but this is highly specialized code.
66
+ # 2. All productions will have a line containing only { (with optional comment)
67
+ # 3. All productions will end with a line containly only } followed by break in ass 1.
68
+ def translate_action
69
+ line = read
70
+ return false if end_of_actions?(line) || line !~ /case\s+(\d+):/
71
+ case_number = $1.to_i
72
+
73
+ line = read
74
+ return false if line !~ /line\s+(\d+)/
75
+ line_number = $1
76
+
77
+ # Extra boiler plate '{' that we do not need
78
+ line = read
79
+ return false if line !~ /^\s*\{\s*(\/\*.*\*\/)?$/
80
+
81
+ @max_case_number = case_number if case_number > @max_case_number
82
+
83
+ label = "case#{case_number}_line#{line_number}"
84
+
85
+ body = []
86
+ last_line = nil
87
+ while (line = read)
88
+ if line =~ /^\s*\}\s*$/ # Extra trailing boiler plate
89
+ next_line = read
90
+ if next_line =~ /break;/
91
+ break
92
+ else
93
+ body << line
94
+ unread next_line
95
+ end
96
+ else
97
+ body << line
98
+ end
99
+ end
100
+
101
+ @case_bodies[case_number] = body
102
+ true
103
+ end
104
+ end
105
+
106
+ PostProcessor.new(ARGV.shift).translate
@@ -0,0 +1,105 @@
1
+ def get_numbers_until_end_block(table)
2
+ while gets
3
+ break if /\};/ =~ $_
4
+ next if /^\/\// =~ $_
5
+ split(/,/).each do |number|
6
+ n = number.strip
7
+ table.push(n.to_i) unless n == ""
8
+ end
9
+ end
10
+ table
11
+ end
12
+
13
+ while gets
14
+ break if /protected static final short\[\] yyTable = \{/ =~ $_
15
+ print $_
16
+ end
17
+
18
+ # A little hacky...gets before ARGV to shift off and open file
19
+ yytable_prefix = ARGV.shift || ''
20
+
21
+ table4 = get_numbers_until_end_block([])
22
+
23
+ puts " protected static final short[] yyTable = #{yytable_prefix}YyTables.yyTable();"
24
+
25
+ while gets
26
+ break if /protected static final short\[\] yyCheck = \{/ =~ $_
27
+ print $_
28
+ end
29
+
30
+ check4 = get_numbers_until_end_block([])
31
+
32
+ puts " protected static final short[] yyCheck = #{yytable_prefix}YyTables.yyCheck();"
33
+
34
+ while gets
35
+ print $_
36
+ end
37
+
38
+ table2 = table4.slice!(0, table4.size / 2)
39
+ table3 = table4.slice!(0, table4.size / 2)
40
+ table1 = table2.slice!(0, table2.size / 2)
41
+ check2 = check4.slice!(0, check4.size / 2)
42
+ check3 = check4.slice!(0, check4.size / 2)
43
+ check1 = check2.slice!(0, check2.size / 2)
44
+
45
+ def printShortArray(table, f)
46
+ table.each_with_index { |e, i|
47
+ f.print "\n " if (i % 10 == 0)
48
+ begin
49
+ f.printf "%4d, ", e
50
+ rescue ArgumentError => a
51
+ $stderr.puts "Trouble printing '#{e}' on index #{i}"
52
+ end
53
+ }
54
+ end
55
+
56
+ def printShortMethod(f, table, name)
57
+ f.puts " private static final short[] yy#{name}() {"
58
+ f.puts " return new short[] {"
59
+ printShortArray table, f
60
+ f.puts
61
+ f.puts " };"
62
+ f.puts " }"
63
+ f.puts
64
+ end
65
+
66
+ open("#{yytable_prefix}YyTables.java", "w") { |f|
67
+ f.print <<END
68
+ package org.jrubyparser.parser;
69
+
70
+ public class #{yytable_prefix}YyTables {
71
+ private static short[] combine(short[] t1, short[] t2,
72
+ short[] t3, short[] t4) {
73
+ short[] t = new short[t1.length + t2.length + t3.length + t4.length];
74
+ int index = 0;
75
+ System.arraycopy(t1, 0, t, index, t1.length);
76
+ index += t1.length;
77
+ System.arraycopy(t2, 0, t, index, t2.length);
78
+ index += t2.length;
79
+ System.arraycopy(t3, 0, t, index, t3.length);
80
+ index += t3.length;
81
+ System.arraycopy(t4, 0, t, index, t4.length);
82
+ return t;
83
+ }
84
+
85
+ public static final short[] yyTable() {
86
+ return combine(yyTable1(), yyTable2(), yyTable3(), yyTable4());
87
+ }
88
+
89
+ public static final short[] yyCheck() {
90
+ return combine(yyCheck1(), yyCheck2(), yyCheck3(), yyCheck4());
91
+ }
92
+ END
93
+
94
+ printShortMethod(f, table1, "Table1")
95
+ printShortMethod(f, table2, "Table2")
96
+ printShortMethod(f, table3, "Table3")
97
+ printShortMethod(f, table4, "Table4")
98
+
99
+ printShortMethod(f, check1, "Check1")
100
+ printShortMethod(f, check2, "Check2")
101
+ printShortMethod(f, check3, "Check3")
102
+ printShortMethod(f, check4, "Check4")
103
+
104
+ f.puts "}"
105
+ }
@@ -0,0 +1,10 @@
1
+ require 'jruby'
2
+
3
+ class Array
4
+ def to_ast_node(position = nil)
5
+ inject(org.jrubyparser.ast.ArrayNode.new(position)) do |array, value|
6
+ value = value.to_ast_node if value.respond_to? :to_ast_node
7
+ array << value
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,6 @@
1
+ require 'java'
2
+ require 'jruby-parser/util/coercer'
3
+
4
+ class org::jrubyparser::ast::AttrAssignNode
5
+ include JRubyParser::Receiver, JRubyParser::Args
6
+ end
@@ -0,0 +1,11 @@
1
+ class TrueClass
2
+ def to_ast_node(position=nil)
3
+ org.jrubyparser.ast.TrueNode.new(position)
4
+ end
5
+ end
6
+
7
+ class FalseClass
8
+ def to_ast_node(position=nil)
9
+ org.jrubyparser.ast.FalseNode.new(position)
10
+ end
11
+ end
@@ -0,0 +1,6 @@
1
+ require 'java'
2
+ require 'jruby-parser/util/coercer'
3
+
4
+ class org::jrubyparser::ast::CallNode
5
+ include JRubyParser::Receiver, JRubyParser::Value, JRubyParser::Args
6
+ end
@@ -0,0 +1,6 @@
1
+ require 'java'
2
+ require 'jruby-parser/util/coercer'
3
+
4
+ class org::jrubyparser::ast::FCallNode
5
+ include JRubyParser::Args
6
+ end
@@ -0,0 +1,10 @@
1
+ require 'java'
2
+ require 'jruby-parser/util/coercer'
3
+
4
+ class org::jrubyparser::ast::ListNode
5
+ def <<(value)
6
+ value = value.to_ast_node if value.respond_to? :to_ast_node
7
+ value.position = getPosition unless value.position
8
+ add(value)
9
+ end
10
+ end
@@ -0,0 +1,7 @@
1
+ require 'java'
2
+
3
+ class NilClass
4
+ def to_ast_node(position=nil)
5
+ org.jrubyparser.ast.NilNode.new(position)
6
+ end
7
+ end
@@ -0,0 +1,66 @@
1
+ require 'java'
2
+
3
+ class org::jrubyparser::ast::Node
4
+ include Enumerable
5
+
6
+ ##
7
+ # Find nth child element of this node
8
+ #
9
+ def [](value)
10
+ child_nodes[value]
11
+ end
12
+
13
+ ##
14
+ # Replace the nth child element of this node with the specified value.
15
+ # if the value is an actual Ruby value it will attempt to call to_ast_node
16
+ # on it to do automatic coercion to an AST node. If the node does not
17
+ # contain positioning information then it will just use the
18
+ # old nodes information
19
+ def []=(index, value)
20
+ value = value.to_ast_node if value.respond_to? :to_ast_node
21
+
22
+ old_value = child_nodes[index]
23
+ value.position = old_value.position unless value.position
24
+ child_nodes[index] = value
25
+ end
26
+
27
+ ##
28
+ # Find first node by name (which is short_name of actual node)
29
+ # === parameters
30
+ # * _name_ is the name of the class you want to find
31
+ # === examples
32
+ #
33
+ # root.find(:fcall) # Find first child node of type fcall (depth-first)
34
+ #
35
+ def find_type(name, &block)
36
+ name = name.to_s
37
+ return self if name == short_name && (!block_given? || yield(self))
38
+
39
+ child_nodes.each do |child|
40
+ value = child.find_type(name, &block)
41
+
42
+ return value if value
43
+ end
44
+ nil
45
+ end
46
+ alias find_node find_type
47
+
48
+ def each(&block)
49
+ yield self
50
+ child_nodes.each { |child| child.each(&block) }
51
+ end
52
+
53
+ ##
54
+ # Convert this node back to human-readable source code.
55
+ #
56
+ def to_source(opts = {})
57
+ filename = opts[:filename] ? opts[:filename] : '(string)'
58
+ java.io.StringWriter.new.tap do |writer|
59
+ accept org.jrubyparser.rewriter.ReWriteVisitor.new(writer, filename)
60
+ end.to_s
61
+ end
62
+
63
+ def short_name
64
+ java_class.name.gsub(/(^.*\.|Node$)/, '').downcase
65
+ end
66
+ end
@@ -0,0 +1,13 @@
1
+ require 'java'
2
+
3
+ class Fixnum
4
+ def to_ast_node(position=nil)
5
+ org.jrubyparser.ast.FixnumNode.new(position, self)
6
+ end
7
+ end
8
+
9
+ class Float
10
+ def to_ast_node(position=nil)
11
+ org.jrubyparser.ast.FloatNode.new(position, self)
12
+ end
13
+ end
@@ -0,0 +1,6 @@
1
+ require 'java'
2
+ require 'jruby-parser/util/coercer'
3
+
4
+ class org::jrubyparser::ast::OpElementAsgnNode
5
+ include JRubyParser::Receiver, JRubyParser::Value, JRubyParser::Args
6
+ end
@@ -0,0 +1,7 @@
1
+ require 'java'
2
+
3
+ class String
4
+ def to_ast_node(position=nil)
5
+ org.jrubyparser.ast.StrNode.new(position, self.to_java(:string))
6
+ end
7
+ end
@@ -0,0 +1,6 @@
1
+ require 'java'
2
+ require 'jruby-parser/util/coercer'
3
+
4
+ class org::jrubyparser::ast::SuperNode
5
+ include JRubyParser::Args
6
+ end
@@ -0,0 +1,44 @@
1
+ module JRubyParser
2
+ module Receiver
3
+ def self.included(cls)
4
+ cls.class_eval do
5
+ def receiver=(value)
6
+ value = value.to_ast_node if value.respond_to? :to_ast_node
7
+ old_value = getReceiver
8
+ value.position = old_value.position unless value.position
9
+ setReceiver(value)
10
+ end
11
+ end
12
+ end
13
+ end
14
+
15
+ module Value
16
+ def self.included(cls)
17
+ cls.class_eval do
18
+ def value=(value)
19
+ value = value.to_ast_node if value.respond_to? :to_ast_node
20
+ old_value = getValue
21
+ value.position = old_value.position unless value.position
22
+ setValue(value)
23
+ end
24
+ end
25
+ end
26
+ end
27
+
28
+ module Args
29
+ def self.included(cls)
30
+ cls.class_eval do
31
+ def args=(value)
32
+ value = value.to_ast_node if value.respond_to? :to_ast_node
33
+ old_value = getArgs
34
+ unless value.position
35
+ value.position = old_value.position
36
+ value.each { |e| e.position = old_value.position } #if value.respond_to? :each
37
+ end
38
+
39
+ setArgs(value)
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,3 @@
1
+ module JRubyParser
2
+ VERSION = "0.3"
3
+ end
Binary file
@@ -0,0 +1,36 @@
1
+ require 'java'
2
+ require 'jruby-parser.jar'
3
+ require 'jruby-parser/core_ext/array'
4
+ require 'jruby-parser/core_ext/boolean'
5
+ require 'jruby-parser/core_ext/nil'
6
+ require 'jruby-parser/core_ext/call_node'
7
+ require 'jruby-parser/core_ext/fcall_node'
8
+ require 'jruby-parser/core_ext/list_node'
9
+ require 'jruby-parser/core_ext/node'
10
+ require 'jruby-parser/core_ext/numeric' # float,fixnum
11
+ require 'jruby-parser/core_ext/op_element_asgn_node'
12
+ require 'jruby-parser/core_ext/string'
13
+
14
+ module JRubyParser
15
+ Compat = org.jrubyparser.CompatVersion
16
+
17
+ ##
18
+ # Parse source string and return a Abstract Syntax Tree (AST) of the source.
19
+ # You may also pass in additional options to affect the reported filename
20
+ # and which version of Ruby you want to use:
21
+ #
22
+ # === Parameters
23
+ # * _source_string_ source you want to parse
24
+ # * _opts_ customize how your source is parsed (:filename, and :version [defaults to 1.9])
25
+ # === Example
26
+ # JRubyParser.parse(%q{puts "hello world"}, :version => JRubyParser::Compat::RUBY1_8)
27
+ #
28
+ def parse(source_string, opts={})
29
+ filename = opts[:filename] ? opts[:filename] : '(string)'
30
+ version = opts[:version] ? opts[:version] : Compat::RUBY1_9
31
+ config = org.jrubyparser.parser.ParserConfiguration.new(0, version)
32
+ reader = java.io.StringReader.new(source_string)
33
+ org.jrubyparser.Parser.new.parse(filename, reader, config)
34
+ end
35
+ module_function :parse
36
+ end
data/lib/yydebug.jar ADDED
Binary file
data/sample/simple.rb ADDED
@@ -0,0 +1,20 @@
1
+ require 'jruby-parser'
2
+
3
+ # Extra options hash can be passed in to override parser configuration
4
+ # opts = {:version => JRubyParser::Compat::RUBY1_8, :filename => 'name.rb'}
5
+ # root = JRubyParser.parse("b = foo(1)", opts)
6
+ root = JRubyParser.parse("b = foo(1)")
7
+
8
+ # Enumerable is mixed into AST tree
9
+ # fcall = foot.find { |e| e.short_name == "fcall" }
10
+ # ...but find_node is pretty common for spec writing:
11
+ fcall = root.find_node(:fcall)
12
+
13
+ # Change the AST.
14
+ fcall.name = 'bar'
15
+
16
+ # Notice this should be a TrueNode, but true knows to coerce into TrueNode
17
+ fcall.args[0] = true
18
+
19
+ # Write out the new source "b = bar(1)"
20
+ p root.to_source
@@ -0,0 +1,20 @@
1
+ $LOAD_PATH.unshift File.dirname(__FILE__) + "/../helpers"
2
+ $LOAD_PATH.unshift File.dirname(__FILE__) + "/../../lib"
3
+ require 'jruby-parser'
4
+ require 'parser_helpers'
5
+ require 'node_helpers'
6
+
7
+ describe Parser do
8
+ it "should parse alias with quotationmarks" do
9
+ root = parse <<-EOF
10
+ def foo(arg)
11
+ puts arg
12
+ end
13
+ EOF
14
+ list = root.pathTo(root.find_node(:defn))
15
+ list.size.should == 3
16
+ list.root.node_name.should == "RootNode"
17
+ list.leaf.node_name.should == "DefnNode"
18
+ end
19
+ end
20
+
@@ -0,0 +1,120 @@
1
+ require 'java'
2
+
3
+ import org.jrubyparser.SourcePosition
4
+ import org.jrubyparser.ast.Node
5
+
6
+ class SourcePosition
7
+ def to_a
8
+ [start_line, end_line, start_offset, end_offset]
9
+ end
10
+ end
11
+
12
+ ###########
13
+
14
+ class AstPositionMatcher
15
+ def initialize(*args)
16
+ @position = args
17
+ end
18
+
19
+ def matches?(actual)
20
+ @actual = actual
21
+ actual.position.to_a == @position
22
+ end
23
+
24
+ def failure_message
25
+ return %[expected #{@actual.position.to_a.inspect} to have position #{@position.inspect}]
26
+ end
27
+
28
+ def negative_failure_message
29
+ return %[expected #{@actual.position.to_a.inspect} to not have position #{@position.inspect}]
30
+ end
31
+ end
32
+
33
+ module HavePosition
34
+ def have_position(*args)
35
+ AstPositionMatcher.new(*args)
36
+ end
37
+ end
38
+
39
+ class AstNameMatcher
40
+ def initialize(name)
41
+ @name = name
42
+ end
43
+
44
+ def matches?(actual)
45
+ @actual = actual
46
+ actual.name == @name
47
+ end
48
+
49
+ def failure_message
50
+ return %[expected #{@actual.inspect} to have name #{@name.inspect}]
51
+ end
52
+
53
+ def negative_failure_message
54
+ return %[expected #{@actual.inspect} to not have name #{@name.inspect}]
55
+ end
56
+ end
57
+
58
+ module HaveName
59
+ def have_name(name)
60
+ AstNameMatcher.new(name)
61
+ end
62
+ end
63
+
64
+ class AstNameAndPositionMatcher
65
+ def initialize(name, *args)
66
+ @name, @position = name, args
67
+ end
68
+
69
+ def matches?(actual)
70
+ @actual = actual
71
+ actual.name == @name && actual.position.to_a == @position
72
+ end
73
+
74
+ def failure_message
75
+ return %[expected #{@actual.name.inspect}, #{@actual.position.to_a.inspect} to have name and position #{@name.inspect}, #{@position.inspect}]
76
+ end
77
+
78
+ def negative_failure_message
79
+ return %[expected #{@actual.name.inspect}, #{@actual.position.to_a.inspect} to not have and position #{@name.inspect}, #{@position.inspect}]
80
+ end
81
+ end
82
+
83
+ module HaveNameAndPosition
84
+ def have_name_and_position(*args)
85
+ AstNameAndPositionMatcher.new(*args)
86
+ end
87
+ end
88
+
89
+ class ArgCountsMatcher
90
+ def initialize(pre, optional, rest, block)
91
+ @args = [pre, optional, rest, block]
92
+ end
93
+
94
+ def matches?(args)
95
+ @actual = [args.pre_count, args.optional_count, args.rest != nil, args.block != nil]
96
+ @args == @actual
97
+ end
98
+
99
+ def failure_message
100
+ return %[expected #{@actual.inspect} to have name #{@args.inspect}]
101
+ end
102
+
103
+ def negative_failure_message
104
+ return %[expected #{@actual.inspect} to not have name #{@args.inspect}]
105
+ end
106
+ end
107
+
108
+ module HaveArgCounts
109
+ def have_arg_counts(*args)
110
+ ArgCountsMatcher.new(*args)
111
+ end
112
+ end
113
+
114
+ #module Spec::Example::ExampleMethods
115
+ class Object
116
+ include HavePosition
117
+ include HaveName
118
+ include HaveNameAndPosition
119
+ include HaveArgCounts
120
+ end