personify 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/LICENSE +20 -0
- data/README.md +172 -0
- data/Rakefile +53 -0
- data/VERSION +1 -0
- data/doc/syntax_ideas.md +141 -0
- data/lib/personify/context.rb +55 -0
- data/lib/personify/parser/personify.rb +1071 -0
- data/lib/personify/parser/personify.treetop +107 -0
- data/lib/personify/parser/personify_node_classes.rb +121 -0
- data/lib/personify/template.rb +17 -0
- data/lib/personify.rb +8 -0
- data/script/generate_parser.rb +6 -0
- data/test/context_test.rb +122 -0
- data/test/fixtures/multiple_tags.txt +8 -0
- data/test/parse_runner.rb +60 -0
- data/test/parser_test.rb +291 -0
- data/test/test_helper.rb +16 -0
- data/vendor/treetop/.gitignore +5 -0
- data/vendor/treetop/History.txt +9 -0
- data/vendor/treetop/README +164 -0
- data/vendor/treetop/Rakefile +20 -0
- data/vendor/treetop/Treetop.tmbundle/Snippets/grammar ___ end.tmSnippet +20 -0
- data/vendor/treetop/Treetop.tmbundle/Snippets/rule ___ end.tmSnippet +18 -0
- data/vendor/treetop/Treetop.tmbundle/Syntaxes/Treetop Grammar.tmLanguage +251 -0
- data/vendor/treetop/Treetop.tmbundle/info.plist +10 -0
- data/vendor/treetop/bin/tt +28 -0
- data/vendor/treetop/doc/contributing_and_planned_features.markdown +103 -0
- data/vendor/treetop/doc/grammar_composition.markdown +65 -0
- data/vendor/treetop/doc/index.markdown +90 -0
- data/vendor/treetop/doc/pitfalls_and_advanced_techniques.markdown +51 -0
- data/vendor/treetop/doc/semantic_interpretation.markdown +189 -0
- data/vendor/treetop/doc/site.rb +110 -0
- data/vendor/treetop/doc/sitegen.rb +60 -0
- data/vendor/treetop/doc/syntactic_recognition.markdown +100 -0
- data/vendor/treetop/doc/using_in_ruby.markdown +21 -0
- data/vendor/treetop/examples/lambda_calculus/arithmetic.rb +551 -0
- data/vendor/treetop/examples/lambda_calculus/arithmetic.treetop +97 -0
- data/vendor/treetop/examples/lambda_calculus/arithmetic_node_classes.rb +7 -0
- data/vendor/treetop/examples/lambda_calculus/arithmetic_test.rb +54 -0
- data/vendor/treetop/examples/lambda_calculus/lambda_calculus +0 -0
- data/vendor/treetop/examples/lambda_calculus/lambda_calculus.rb +718 -0
- data/vendor/treetop/examples/lambda_calculus/lambda_calculus.treetop +132 -0
- data/vendor/treetop/examples/lambda_calculus/lambda_calculus_node_classes.rb +5 -0
- data/vendor/treetop/examples/lambda_calculus/lambda_calculus_test.rb +89 -0
- data/vendor/treetop/examples/lambda_calculus/test_helper.rb +18 -0
- data/vendor/treetop/lib/treetop/bootstrap_gen_1_metagrammar.rb +45 -0
- data/vendor/treetop/lib/treetop/compiler/grammar_compiler.rb +40 -0
- data/vendor/treetop/lib/treetop/compiler/lexical_address_space.rb +17 -0
- data/vendor/treetop/lib/treetop/compiler/metagrammar.rb +2955 -0
- data/vendor/treetop/lib/treetop/compiler/metagrammar.treetop +404 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/anything_symbol.rb +20 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/atomic_expression.rb +14 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/character_class.rb +22 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/choice.rb +31 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/declaration_sequence.rb +24 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/grammar.rb +28 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/inline_module.rb +27 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/nonterminal.rb +13 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/optional.rb +19 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/parenthesized_expression.rb +9 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/parsing_expression.rb +138 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/parsing_rule.rb +55 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/predicate.rb +45 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/repetition.rb +55 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/sequence.rb +68 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/terminal.rb +20 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/transient_prefix.rb +9 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes/treetop_file.rb +9 -0
- data/vendor/treetop/lib/treetop/compiler/node_classes.rb +19 -0
- data/vendor/treetop/lib/treetop/compiler/ruby_builder.rb +113 -0
- data/vendor/treetop/lib/treetop/compiler.rb +6 -0
- data/vendor/treetop/lib/treetop/ruby_extensions/string.rb +42 -0
- data/vendor/treetop/lib/treetop/ruby_extensions.rb +2 -0
- data/vendor/treetop/lib/treetop/runtime/compiled_parser.rb +95 -0
- data/vendor/treetop/lib/treetop/runtime/interval_skip_list/head_node.rb +15 -0
- data/vendor/treetop/lib/treetop/runtime/interval_skip_list/interval_skip_list.rb +200 -0
- data/vendor/treetop/lib/treetop/runtime/interval_skip_list/node.rb +164 -0
- data/vendor/treetop/lib/treetop/runtime/interval_skip_list.rb +4 -0
- data/vendor/treetop/lib/treetop/runtime/syntax_node.rb +72 -0
- data/vendor/treetop/lib/treetop/runtime/terminal_parse_failure.rb +16 -0
- data/vendor/treetop/lib/treetop/runtime/terminal_syntax_node.rb +17 -0
- data/vendor/treetop/lib/treetop/runtime.rb +5 -0
- data/vendor/treetop/lib/treetop/version.rb +9 -0
- data/vendor/treetop/lib/treetop.rb +11 -0
- data/vendor/treetop/script/generate_metagrammar.rb +14 -0
- data/vendor/treetop/script/svnadd +11 -0
- data/vendor/treetop/script/svnrm +11 -0
- data/vendor/treetop/spec/compiler/and_predicate_spec.rb +36 -0
- data/vendor/treetop/spec/compiler/anything_symbol_spec.rb +52 -0
- data/vendor/treetop/spec/compiler/character_class_spec.rb +188 -0
- data/vendor/treetop/spec/compiler/choice_spec.rb +80 -0
- data/vendor/treetop/spec/compiler/circular_compilation_spec.rb +28 -0
- data/vendor/treetop/spec/compiler/failure_propagation_functional_spec.rb +21 -0
- data/vendor/treetop/spec/compiler/grammar_compiler_spec.rb +84 -0
- data/vendor/treetop/spec/compiler/grammar_spec.rb +41 -0
- data/vendor/treetop/spec/compiler/nonterminal_symbol_spec.rb +40 -0
- data/vendor/treetop/spec/compiler/not_predicate_spec.rb +38 -0
- data/vendor/treetop/spec/compiler/one_or_more_spec.rb +35 -0
- data/vendor/treetop/spec/compiler/optional_spec.rb +37 -0
- data/vendor/treetop/spec/compiler/parenthesized_expression_spec.rb +19 -0
- data/vendor/treetop/spec/compiler/parsing_rule_spec.rb +32 -0
- data/vendor/treetop/spec/compiler/sequence_spec.rb +115 -0
- data/vendor/treetop/spec/compiler/terminal_spec.rb +81 -0
- data/vendor/treetop/spec/compiler/terminal_symbol_spec.rb +37 -0
- data/vendor/treetop/spec/compiler/test_grammar.treetop +7 -0
- data/vendor/treetop/spec/compiler/test_grammar.tt +7 -0
- data/vendor/treetop/spec/compiler/test_grammar_do.treetop +7 -0
- data/vendor/treetop/spec/compiler/zero_or_more_spec.rb +56 -0
- data/vendor/treetop/spec/composition/a.treetop +11 -0
- data/vendor/treetop/spec/composition/b.treetop +11 -0
- data/vendor/treetop/spec/composition/c.treetop +10 -0
- data/vendor/treetop/spec/composition/d.treetop +10 -0
- data/vendor/treetop/spec/composition/grammar_composition_spec.rb +26 -0
- data/vendor/treetop/spec/ruby_extensions/string_spec.rb +32 -0
- data/vendor/treetop/spec/runtime/compiled_parser_spec.rb +101 -0
- data/vendor/treetop/spec/runtime/interval_skip_list/delete_spec.rb +147 -0
- data/vendor/treetop/spec/runtime/interval_skip_list/expire_range_spec.rb +349 -0
- data/vendor/treetop/spec/runtime/interval_skip_list/insert_and_delete_node.rb +385 -0
- data/vendor/treetop/spec/runtime/interval_skip_list/insert_spec.rb +660 -0
- data/vendor/treetop/spec/runtime/interval_skip_list/interval_skip_list_spec.graffle +6175 -0
- data/vendor/treetop/spec/runtime/interval_skip_list/interval_skip_list_spec.rb +58 -0
- data/vendor/treetop/spec/runtime/interval_skip_list/palindromic_fixture.rb +23 -0
- data/vendor/treetop/spec/runtime/interval_skip_list/palindromic_fixture_spec.rb +164 -0
- data/vendor/treetop/spec/runtime/interval_skip_list/spec_helper.rb +84 -0
- data/vendor/treetop/spec/runtime/syntax_node_spec.rb +53 -0
- data/vendor/treetop/spec/spec_helper.rb +106 -0
- data/vendor/treetop/spec/spec_suite.rb +4 -0
- data/vendor/treetop/treetop.gemspec +18 -0
- metadata +196 -0
@@ -0,0 +1,138 @@
|
|
1
|
+
module Treetop
|
2
|
+
module Compiler
|
3
|
+
class ParsingExpression < Runtime::SyntaxNode
|
4
|
+
attr_reader :address, :builder, :subexpression_address, :var_symbols, :parent_expression
|
5
|
+
|
6
|
+
def compile(address, builder, parent_expression)
|
7
|
+
@address = address
|
8
|
+
@builder = builder
|
9
|
+
@parent_expression = parent_expression
|
10
|
+
end
|
11
|
+
|
12
|
+
def node_class_name
|
13
|
+
parent_expression && parent_expression.node_class_name || 'SyntaxNode'
|
14
|
+
end
|
15
|
+
|
16
|
+
def declared_module_name
|
17
|
+
parent_expression && parent_expression.node_class_name
|
18
|
+
end
|
19
|
+
|
20
|
+
def inline_module_name
|
21
|
+
parent_expression && parent_expression.inline_module_name
|
22
|
+
end
|
23
|
+
|
24
|
+
def optional_arg(arg)
|
25
|
+
if arg
|
26
|
+
", #{arg}"
|
27
|
+
else
|
28
|
+
''
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def use_vars(*var_symbols)
|
33
|
+
@var_symbols = var_symbols
|
34
|
+
builder << var_initialization
|
35
|
+
end
|
36
|
+
|
37
|
+
def result_var
|
38
|
+
var(:result)
|
39
|
+
end
|
40
|
+
|
41
|
+
def accumulator_var
|
42
|
+
var(:accumulator)
|
43
|
+
end
|
44
|
+
|
45
|
+
def start_index_var
|
46
|
+
var(:start_index)
|
47
|
+
end
|
48
|
+
|
49
|
+
def subexpression_result_var
|
50
|
+
"r#{subexpression_address}"
|
51
|
+
end
|
52
|
+
|
53
|
+
def subexpression_success?
|
54
|
+
subexpression_result_var
|
55
|
+
end
|
56
|
+
|
57
|
+
def obtain_new_subexpression_address
|
58
|
+
@subexpression_address = builder.next_address
|
59
|
+
end
|
60
|
+
|
61
|
+
def accumulate_subexpression_result
|
62
|
+
builder.accumulate accumulator_var, subexpression_result_var
|
63
|
+
end
|
64
|
+
|
65
|
+
def assign_result(value_ruby)
|
66
|
+
builder.assign result_var, value_ruby
|
67
|
+
end
|
68
|
+
|
69
|
+
def extend_result(module_name)
|
70
|
+
builder.extend result_var, module_name
|
71
|
+
end
|
72
|
+
|
73
|
+
def extend_result_with_declared_module
|
74
|
+
extend_result declared_module_name if declared_module_name
|
75
|
+
end
|
76
|
+
|
77
|
+
def extend_result_with_inline_module
|
78
|
+
extend_result inline_module_name if inline_module_name
|
79
|
+
end
|
80
|
+
|
81
|
+
def reset_index
|
82
|
+
builder.assign 'self.index', start_index_var
|
83
|
+
end
|
84
|
+
|
85
|
+
def epsilon_node
|
86
|
+
"instantiate_node(SyntaxNode,input, index...index)"
|
87
|
+
end
|
88
|
+
|
89
|
+
def assign_failure(start_index_var)
|
90
|
+
assign_result("nil")
|
91
|
+
end
|
92
|
+
|
93
|
+
def var_initialization
|
94
|
+
left, right = [], []
|
95
|
+
var_symbols.each do |symbol|
|
96
|
+
if init_value(symbol)
|
97
|
+
left << var(symbol)
|
98
|
+
right << init_value(symbol)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
if left.empty?
|
102
|
+
""
|
103
|
+
else
|
104
|
+
left.join(', ') + ' = ' + right.join(', ')
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def var(var_symbol)
|
109
|
+
case var_symbol
|
110
|
+
when :result then "r#{address}"
|
111
|
+
when :accumulator then "s#{address}"
|
112
|
+
when :start_index then "i#{address}"
|
113
|
+
else raise "Unknown var symbol #{var_symbol}."
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def init_value(var_symbol)
|
118
|
+
case var_symbol
|
119
|
+
when :accumulator then '[]'
|
120
|
+
when :start_index then 'index'
|
121
|
+
else nil
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def begin_comment(expression)
|
126
|
+
#builder << "# begin #{on_one_line(expression)}"
|
127
|
+
end
|
128
|
+
|
129
|
+
def end_comment(expression)
|
130
|
+
#builder << "# end #{on_one_line(expression)}"
|
131
|
+
end
|
132
|
+
|
133
|
+
def on_one_line(expression)
|
134
|
+
expression.text_value.tr("\n", ' ')
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module Treetop
|
2
|
+
module Compiler
|
3
|
+
class ParsingRule < Runtime::SyntaxNode
|
4
|
+
|
5
|
+
def compile(builder)
|
6
|
+
compile_inline_module_declarations(builder)
|
7
|
+
generate_method_definition(builder)
|
8
|
+
end
|
9
|
+
|
10
|
+
def compile_inline_module_declarations(builder)
|
11
|
+
parsing_expression.inline_modules.each_with_index do |inline_module, i|
|
12
|
+
inline_module.compile(i, builder, self)
|
13
|
+
builder.newline
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def generate_method_definition(builder)
|
18
|
+
builder.reset_addresses
|
19
|
+
expression_address = builder.next_address
|
20
|
+
result_var = "r#{expression_address}"
|
21
|
+
|
22
|
+
builder.method_declaration(method_name) do
|
23
|
+
builder.assign 'start_index', 'index'
|
24
|
+
generate_cache_lookup(builder)
|
25
|
+
builder.newline
|
26
|
+
parsing_expression.compile(expression_address, builder)
|
27
|
+
builder.newline
|
28
|
+
generate_cache_storage(builder, result_var)
|
29
|
+
builder.newline
|
30
|
+
builder << "return #{result_var}"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def generate_cache_lookup(builder)
|
35
|
+
builder.if_ "node_cache[:#{name}].has_key?(index)" do
|
36
|
+
builder.assign 'cached', "node_cache[:#{name}][index]"
|
37
|
+
builder << '@index = cached.interval.end if cached'
|
38
|
+
builder << 'return cached'
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def generate_cache_storage(builder, result_var)
|
43
|
+
builder.assign "node_cache[:#{name}][start_index]", result_var
|
44
|
+
end
|
45
|
+
|
46
|
+
def method_name
|
47
|
+
"_nt_#{name}"
|
48
|
+
end
|
49
|
+
|
50
|
+
def name
|
51
|
+
nonterminal.text_value
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module Treetop
|
2
|
+
module Compiler
|
3
|
+
class Predicate < ParsingExpression
|
4
|
+
def compile(address, builder, parent_expression)
|
5
|
+
super
|
6
|
+
begin_comment(parent_expression)
|
7
|
+
use_vars :result, :start_index
|
8
|
+
obtain_new_subexpression_address
|
9
|
+
parent_expression.prefixed_expression.compile(subexpression_address, builder)
|
10
|
+
builder.if__(subexpression_success?) { when_success }
|
11
|
+
builder.else_ { when_failure }
|
12
|
+
end_comment(parent_expression)
|
13
|
+
end
|
14
|
+
|
15
|
+
def assign_failure
|
16
|
+
super(start_index_var)
|
17
|
+
end
|
18
|
+
|
19
|
+
def assign_success
|
20
|
+
reset_index
|
21
|
+
assign_result epsilon_node
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
class AndPredicate < Predicate
|
26
|
+
def when_success
|
27
|
+
assign_success
|
28
|
+
end
|
29
|
+
|
30
|
+
def when_failure
|
31
|
+
assign_failure
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
class NotPredicate < Predicate
|
36
|
+
def when_success
|
37
|
+
assign_failure
|
38
|
+
end
|
39
|
+
|
40
|
+
def when_failure
|
41
|
+
assign_success
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module Treetop
|
2
|
+
module Compiler
|
3
|
+
class Repetition < ParsingExpression
|
4
|
+
def compile(address, builder, parent_expression)
|
5
|
+
super
|
6
|
+
repeated_expression = parent_expression.atomic
|
7
|
+
begin_comment(parent_expression)
|
8
|
+
use_vars :result, :accumulator, :start_index
|
9
|
+
|
10
|
+
builder.loop do
|
11
|
+
obtain_new_subexpression_address
|
12
|
+
repeated_expression.compile(subexpression_address, builder)
|
13
|
+
builder.if__ subexpression_success? do
|
14
|
+
accumulate_subexpression_result
|
15
|
+
end
|
16
|
+
builder.else_ do
|
17
|
+
builder.break
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def inline_module_name
|
23
|
+
parent_expression.inline_module_name
|
24
|
+
end
|
25
|
+
|
26
|
+
def assign_and_extend_result
|
27
|
+
assign_result "instantiate_node(#{node_class_name},input, #{start_index_var}...index, #{accumulator_var})"
|
28
|
+
extend_result_with_inline_module
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
|
33
|
+
class ZeroOrMore < Repetition
|
34
|
+
def compile(address, builder, parent_expression)
|
35
|
+
super
|
36
|
+
assign_and_extend_result
|
37
|
+
end_comment(parent_expression)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
class OneOrMore < Repetition
|
42
|
+
def compile(address, builder, parent_expression)
|
43
|
+
super
|
44
|
+
builder.if__ "#{accumulator_var}.empty?" do
|
45
|
+
reset_index
|
46
|
+
assign_failure start_index_var
|
47
|
+
end
|
48
|
+
builder.else_ do
|
49
|
+
assign_and_extend_result
|
50
|
+
end
|
51
|
+
end_comment(parent_expression)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
module Treetop
|
2
|
+
module Compiler
|
3
|
+
class Sequence < ParsingExpression
|
4
|
+
def compile(address, builder, parent_expression = nil)
|
5
|
+
super
|
6
|
+
begin_comment(self)
|
7
|
+
use_vars :result, :start_index, :accumulator
|
8
|
+
compile_sequence_elements(sequence_elements)
|
9
|
+
builder.if__ "#{accumulator_var}.last" do
|
10
|
+
assign_result "instantiate_node(#{node_class_name},input, #{start_index_var}...index, #{accumulator_var})"
|
11
|
+
extend_result sequence_element_accessor_module_name if sequence_element_accessor_module_name
|
12
|
+
extend_result_with_inline_module
|
13
|
+
end
|
14
|
+
builder.else_ do
|
15
|
+
reset_index
|
16
|
+
assign_failure start_index_var
|
17
|
+
end
|
18
|
+
end_comment(self)
|
19
|
+
end
|
20
|
+
|
21
|
+
def node_class_name
|
22
|
+
node_class_declarations.node_class_name || 'SyntaxNode'
|
23
|
+
end
|
24
|
+
|
25
|
+
def compile_sequence_elements(elements)
|
26
|
+
obtain_new_subexpression_address
|
27
|
+
elements.first.compile(subexpression_address, builder)
|
28
|
+
accumulate_subexpression_result
|
29
|
+
if elements.size > 1
|
30
|
+
builder.if_ subexpression_success? do
|
31
|
+
compile_sequence_elements(elements[1..-1])
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def sequence_element_accessor_module
|
37
|
+
@sequence_element_accessor_module ||= SequenceElementAccessorModule.new(sequence_elements)
|
38
|
+
end
|
39
|
+
|
40
|
+
def sequence_element_accessor_module_name
|
41
|
+
sequence_element_accessor_module.module_name
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
class SequenceElementAccessorModule
|
46
|
+
include InlineModuleMixin
|
47
|
+
attr_reader :sequence_elements
|
48
|
+
|
49
|
+
def initialize(sequence_elements)
|
50
|
+
@sequence_elements = sequence_elements
|
51
|
+
end
|
52
|
+
|
53
|
+
def compile(index, builder, rule)
|
54
|
+
super
|
55
|
+
builder.module_declaration(module_name) do
|
56
|
+
sequence_elements.each_with_index do |element, index|
|
57
|
+
if element.label_name
|
58
|
+
builder.method_declaration(element.label_name) do
|
59
|
+
builder << "elements[#{index}]"
|
60
|
+
end
|
61
|
+
builder.newline unless index == sequence_elements.size - 1
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Treetop
|
2
|
+
module Compiler
|
3
|
+
class Terminal < AtomicExpression
|
4
|
+
def compile(address, builder, parent_expression = nil)
|
5
|
+
super
|
6
|
+
string_length = eval(text_value).length
|
7
|
+
|
8
|
+
builder.if__ "input.index(#{text_value}, index) == index" do
|
9
|
+
assign_result "instantiate_node(#{node_class_name},input, index...(index + #{string_length}))"
|
10
|
+
extend_result_with_inline_module
|
11
|
+
builder << "@index += #{string_length}"
|
12
|
+
end
|
13
|
+
builder.else_ do
|
14
|
+
builder << "terminal_parse_failure(#{text_value})"
|
15
|
+
assign_result 'nil'
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
dir = File.dirname(__FILE__)
|
2
|
+
require File.join(dir, *%w[node_classes parsing_expression])
|
3
|
+
require File.join(dir, *%w[node_classes atomic_expression])
|
4
|
+
require File.join(dir, *%w[node_classes inline_module])
|
5
|
+
require File.join(dir, *%w[node_classes treetop_file])
|
6
|
+
require File.join(dir, *%w[node_classes grammar])
|
7
|
+
require File.join(dir, *%w[node_classes declaration_sequence])
|
8
|
+
require File.join(dir, *%w[node_classes parsing_rule])
|
9
|
+
require File.join(dir, *%w[node_classes parenthesized_expression])
|
10
|
+
require File.join(dir, *%w[node_classes nonterminal])
|
11
|
+
require File.join(dir, *%w[node_classes terminal])
|
12
|
+
require File.join(dir, *%w[node_classes anything_symbol])
|
13
|
+
require File.join(dir, *%w[node_classes character_class])
|
14
|
+
require File.join(dir, *%w[node_classes sequence])
|
15
|
+
require File.join(dir, *%w[node_classes choice])
|
16
|
+
require File.join(dir, *%w[node_classes repetition])
|
17
|
+
require File.join(dir, *%w[node_classes optional])
|
18
|
+
require File.join(dir, *%w[node_classes predicate])
|
19
|
+
require File.join(dir, *%w[node_classes transient_prefix])
|
@@ -0,0 +1,113 @@
|
|
1
|
+
module Treetop
|
2
|
+
module Compiler
|
3
|
+
class RubyBuilder
|
4
|
+
|
5
|
+
attr_reader :level, :address_space, :ruby
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
@level = 0
|
9
|
+
@address_space = LexicalAddressSpace.new
|
10
|
+
@ruby = ""
|
11
|
+
end
|
12
|
+
|
13
|
+
def <<(ruby_line)
|
14
|
+
return if ruby_line.blank?
|
15
|
+
ruby << ruby_line.tabto(level) << "\n"
|
16
|
+
end
|
17
|
+
|
18
|
+
def newline
|
19
|
+
ruby << "\n"
|
20
|
+
end
|
21
|
+
|
22
|
+
def indented(depth = 2)
|
23
|
+
self.in(depth)
|
24
|
+
yield
|
25
|
+
self.out(depth)
|
26
|
+
end
|
27
|
+
|
28
|
+
def class_declaration(name, &block)
|
29
|
+
self << "class #{name}"
|
30
|
+
indented(&block)
|
31
|
+
self << "end"
|
32
|
+
end
|
33
|
+
|
34
|
+
def module_declaration(name, &block)
|
35
|
+
self << "module #{name}"
|
36
|
+
indented(&block)
|
37
|
+
self << "end"
|
38
|
+
end
|
39
|
+
|
40
|
+
def method_declaration(name, &block)
|
41
|
+
self << "def #{name}"
|
42
|
+
indented(&block)
|
43
|
+
self << "end"
|
44
|
+
end
|
45
|
+
|
46
|
+
def assign(left, right)
|
47
|
+
if left.instance_of? Array
|
48
|
+
self << "#{left.join(', ')} = #{right.join(', ')}"
|
49
|
+
else
|
50
|
+
self << "#{left} = #{right}"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def extend(var, module_name)
|
55
|
+
self << "#{var}.extend(#{module_name})"
|
56
|
+
end
|
57
|
+
|
58
|
+
def accumulate(left, right)
|
59
|
+
self << "#{left} << #{right}"
|
60
|
+
end
|
61
|
+
|
62
|
+
def if__(condition, &block)
|
63
|
+
self << "if #{condition}"
|
64
|
+
indented(&block)
|
65
|
+
end
|
66
|
+
|
67
|
+
def if_(condition, &block)
|
68
|
+
if__(condition, &block)
|
69
|
+
self << 'end'
|
70
|
+
end
|
71
|
+
|
72
|
+
def else_(&block)
|
73
|
+
self << 'else'
|
74
|
+
indented(&block)
|
75
|
+
self << 'end'
|
76
|
+
end
|
77
|
+
|
78
|
+
def loop(&block)
|
79
|
+
self << 'loop do'
|
80
|
+
indented(&block)
|
81
|
+
self << 'end'
|
82
|
+
end
|
83
|
+
|
84
|
+
def break
|
85
|
+
self << 'break'
|
86
|
+
end
|
87
|
+
|
88
|
+
def in(depth = 2)
|
89
|
+
@level += depth
|
90
|
+
self
|
91
|
+
end
|
92
|
+
|
93
|
+
def out(depth = 2)
|
94
|
+
@level -= depth
|
95
|
+
self
|
96
|
+
end
|
97
|
+
|
98
|
+
def next_address
|
99
|
+
address_space.next_address
|
100
|
+
end
|
101
|
+
|
102
|
+
def reset_addresses
|
103
|
+
address_space.reset_addresses
|
104
|
+
end
|
105
|
+
|
106
|
+
protected
|
107
|
+
|
108
|
+
def indent
|
109
|
+
" " * level
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
@@ -0,0 +1,6 @@
|
|
1
|
+
dir = File.dirname(__FILE__)
|
2
|
+
require File.join(dir, *%w[compiler lexical_address_space])
|
3
|
+
require File.join(dir, *%w[compiler ruby_builder])
|
4
|
+
require File.join(dir, *%w[compiler node_classes])
|
5
|
+
require File.join(dir, *%w[compiler metagrammar]) unless $exclude_metagrammar
|
6
|
+
require File.join(dir, *%w[compiler grammar_compiler])
|
@@ -0,0 +1,42 @@
|
|
1
|
+
class String
|
2
|
+
def column_of(index)
|
3
|
+
return 1 if index == 0
|
4
|
+
newline_index = rindex("\n", index - 1)
|
5
|
+
if newline_index
|
6
|
+
index - newline_index
|
7
|
+
else
|
8
|
+
index + 1
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def line_of(index)
|
13
|
+
self[0...index].count("\n") + 1
|
14
|
+
end
|
15
|
+
|
16
|
+
unless method_defined?(:blank?)
|
17
|
+
def blank?
|
18
|
+
self == ""
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
# The following methods are lifted from Facets 2.0.2
|
23
|
+
def tabto(n)
|
24
|
+
if self =~ /^( *)\S/
|
25
|
+
indent(n - $1.length)
|
26
|
+
else
|
27
|
+
self
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def indent(n)
|
32
|
+
if n >= 0
|
33
|
+
gsub(/^/, ' ' * n)
|
34
|
+
else
|
35
|
+
gsub(/^ {0,#{-n}}/, "")
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def treetop_camelize
|
40
|
+
to_s.gsub(/\/(.?)/){ "::" + $1.upcase }.gsub(/(^|_)(.)/){ $2.upcase }
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
module Treetop
|
2
|
+
module Runtime
|
3
|
+
class CompiledParser
|
4
|
+
include Treetop::Runtime
|
5
|
+
|
6
|
+
attr_reader :input, :index, :terminal_failures, :max_terminal_failure_index
|
7
|
+
attr_writer :root
|
8
|
+
attr_accessor :consume_all_input
|
9
|
+
alias :consume_all_input? :consume_all_input
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
self.consume_all_input = true
|
13
|
+
end
|
14
|
+
|
15
|
+
def parse(input, options = {})
|
16
|
+
prepare_to_parse(input)
|
17
|
+
@index = options[:index] if options[:index]
|
18
|
+
result = send("_nt_#{root}")
|
19
|
+
return nil if (consume_all_input? && index != input.size)
|
20
|
+
return result
|
21
|
+
end
|
22
|
+
|
23
|
+
def failure_index
|
24
|
+
max_terminal_failure_index
|
25
|
+
end
|
26
|
+
|
27
|
+
def failure_line
|
28
|
+
terminal_failures && input.line_of(failure_index)
|
29
|
+
end
|
30
|
+
|
31
|
+
def failure_column
|
32
|
+
terminal_failures && input.column_of(failure_index)
|
33
|
+
end
|
34
|
+
|
35
|
+
def failure_reason
|
36
|
+
return nil unless (tf = terminal_failures) && tf.size > 0
|
37
|
+
"Expected " +
|
38
|
+
(tf.size == 1 ?
|
39
|
+
tf[0].expected_string :
|
40
|
+
"one of #{tf.map{|f| f.expected_string}.uniq*', '}"
|
41
|
+
) +
|
42
|
+
" at line #{failure_line}, column #{failure_column} (byte #{failure_index+1})" +
|
43
|
+
" after #{input[index...failure_index]}"
|
44
|
+
end
|
45
|
+
|
46
|
+
|
47
|
+
protected
|
48
|
+
|
49
|
+
attr_reader :node_cache, :input_length
|
50
|
+
attr_writer :index
|
51
|
+
|
52
|
+
def prepare_to_parse(input)
|
53
|
+
@input = input
|
54
|
+
@input_length = input.length
|
55
|
+
reset_index
|
56
|
+
@node_cache = Hash.new {|hash, key| hash[key] = Hash.new}
|
57
|
+
@terminal_failures = []
|
58
|
+
@max_terminal_failure_index = 0
|
59
|
+
end
|
60
|
+
|
61
|
+
def reset_index
|
62
|
+
@index = 0
|
63
|
+
end
|
64
|
+
|
65
|
+
def parse_anything(node_class = SyntaxNode, inline_module = nil)
|
66
|
+
if index < input.length
|
67
|
+
result = instantiate_node(node_class,input, index...(index + 1))
|
68
|
+
result.extend(inline_module) if inline_module
|
69
|
+
@index += 1
|
70
|
+
result
|
71
|
+
else
|
72
|
+
terminal_parse_failure("any character")
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def instantiate_node(node_type,*args)
|
77
|
+
if node_type.respond_to? :new
|
78
|
+
node_type.new(*args)
|
79
|
+
else
|
80
|
+
SyntaxNode.new(*args).extend(node_type)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def terminal_parse_failure(expected_string)
|
85
|
+
return nil if index < max_terminal_failure_index
|
86
|
+
if index > max_terminal_failure_index
|
87
|
+
@max_terminal_failure_index = index
|
88
|
+
@terminal_failures = []
|
89
|
+
end
|
90
|
+
terminal_failures << TerminalParseFailure.new(index, expected_string)
|
91
|
+
return nil
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|