codeminer 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. checksums.yaml +7 -0
  2. data/codeminer.gemspec +28 -0
  3. data/lib/codeminer.rb +112 -0
  4. data/lib/codeminer/expression_processor.rb +29 -0
  5. data/lib/codeminer/expressions/alias_expression.rb +23 -0
  6. data/lib/codeminer/expressions/ambiguous_operator_expression.rb +27 -0
  7. data/lib/codeminer/expressions/aref_assign_expression.rb +23 -0
  8. data/lib/codeminer/expressions/aref_field_expression.rb +28 -0
  9. data/lib/codeminer/expressions/arguments_expression.rb +29 -0
  10. data/lib/codeminer/expressions/array_expression.rb +22 -0
  11. data/lib/codeminer/expressions/attribute_assign_expression.rb +35 -0
  12. data/lib/codeminer/expressions/backref_variable_expression.rb +22 -0
  13. data/lib/codeminer/expressions/begin_expression.rb +38 -0
  14. data/lib/codeminer/expressions/binary_expression.rb +31 -0
  15. data/lib/codeminer/expressions/block_expression.rb +24 -0
  16. data/lib/codeminer/expressions/block_param_expression.rb +24 -0
  17. data/lib/codeminer/expressions/body_expression.rb +35 -0
  18. data/lib/codeminer/expressions/bodystmt_expression.rb +25 -0
  19. data/lib/codeminer/expressions/break_expression.rb +22 -0
  20. data/lib/codeminer/expressions/call_expression.rb +35 -0
  21. data/lib/codeminer/expressions/case_expression.rb +27 -0
  22. data/lib/codeminer/expressions/class_expression.rb +28 -0
  23. data/lib/codeminer/expressions/class_variable_assign_expression.rb +27 -0
  24. data/lib/codeminer/expressions/class_variable_expression.rb +22 -0
  25. data/lib/codeminer/expressions/colon2_assign_expression.rb +23 -0
  26. data/lib/codeminer/expressions/colon2_expression.rb +27 -0
  27. data/lib/codeminer/expressions/colon3_assign_expression.rb +23 -0
  28. data/lib/codeminer/expressions/colon3_expression.rb +26 -0
  29. data/lib/codeminer/expressions/command_expression.rb +30 -0
  30. data/lib/codeminer/expressions/condition_expression.rb +25 -0
  31. data/lib/codeminer/expressions/constant_assign_expression.rb +27 -0
  32. data/lib/codeminer/expressions/constant_variable_expression.rb +24 -0
  33. data/lib/codeminer/expressions/defined_expression.rb +22 -0
  34. data/lib/codeminer/expressions/defn_expression.rb +24 -0
  35. data/lib/codeminer/expressions/defs_expression.rb +29 -0
  36. data/lib/codeminer/expressions/destructured_param_expression.rb +24 -0
  37. data/lib/codeminer/expressions/dynamic_string_expression.rb +36 -0
  38. data/lib/codeminer/expressions/dynamic_symbol_expression.rb +22 -0
  39. data/lib/codeminer/expressions/else_expression.rb +26 -0
  40. data/lib/codeminer/expressions/empty_expression.rb +19 -0
  41. data/lib/codeminer/expressions/ensure_expression.rb +22 -0
  42. data/lib/codeminer/expressions/expression.rb +24 -0
  43. data/lib/codeminer/expressions/expression_container.rb +41 -0
  44. data/lib/codeminer/expressions/false_expression.rb +22 -0
  45. data/lib/codeminer/expressions/field_expression.rb +27 -0
  46. data/lib/codeminer/expressions/for_expression.rb +24 -0
  47. data/lib/codeminer/expressions/global_variable_assign_expression.rb +27 -0
  48. data/lib/codeminer/expressions/global_variable_expression.rb +22 -0
  49. data/lib/codeminer/expressions/hash_expression.rb +22 -0
  50. data/lib/codeminer/expressions/hash_pair_expression.rb +23 -0
  51. data/lib/codeminer/expressions/instance_variable_assign_expression.rb +27 -0
  52. data/lib/codeminer/expressions/instance_variable_expression.rb +22 -0
  53. data/lib/codeminer/expressions/int_expression.rb +24 -0
  54. data/lib/codeminer/expressions/keyword_param_expression.rb +31 -0
  55. data/lib/codeminer/expressions/lambda_expression.rb +25 -0
  56. data/lib/codeminer/expressions/local_assign_expression.rb +35 -0
  57. data/lib/codeminer/expressions/local_variable_expression.rb +24 -0
  58. data/lib/codeminer/expressions/magic_comment_expression.rb +23 -0
  59. data/lib/codeminer/expressions/module_expression.rb +27 -0
  60. data/lib/codeminer/expressions/multiple_assignment_expression.rb +25 -0
  61. data/lib/codeminer/expressions/next_expression.rb +22 -0
  62. data/lib/codeminer/expressions/nil_expression.rb +22 -0
  63. data/lib/codeminer/expressions/op_assign_expression.rb +26 -0
  64. data/lib/codeminer/expressions/optional_param_expression.rb +31 -0
  65. data/lib/codeminer/expressions/params_expression.rb +67 -0
  66. data/lib/codeminer/expressions/positional_param_expression.rb +32 -0
  67. data/lib/codeminer/expressions/range_expression.rb +28 -0
  68. data/lib/codeminer/expressions/redo_expression.rb +21 -0
  69. data/lib/codeminer/expressions/regexp_expression.rb +26 -0
  70. data/lib/codeminer/expressions/rescue_body_expression.rb +22 -0
  71. data/lib/codeminer/expressions/rescue_expression.rb +25 -0
  72. data/lib/codeminer/expressions/retry_expression.rb +21 -0
  73. data/lib/codeminer/expressions/return_expression.rb +26 -0
  74. data/lib/codeminer/expressions/root_expression.rb +22 -0
  75. data/lib/codeminer/expressions/sclass_expression.rb +23 -0
  76. data/lib/codeminer/expressions/self_expression.rb +22 -0
  77. data/lib/codeminer/expressions/splat_expression.rb +24 -0
  78. data/lib/codeminer/expressions/string_content_expression.rb +29 -0
  79. data/lib/codeminer/expressions/string_embedded_expression.rb +22 -0
  80. data/lib/codeminer/expressions/string_expression.rb +45 -0
  81. data/lib/codeminer/expressions/super_expression.rb +26 -0
  82. data/lib/codeminer/expressions/symbol_expression.rb +32 -0
  83. data/lib/codeminer/expressions/true_expression.rb +22 -0
  84. data/lib/codeminer/expressions/unary_expression.rb +23 -0
  85. data/lib/codeminer/expressions/undef_expression.rb +22 -0
  86. data/lib/codeminer/expressions/unknown_expression.rb +35 -0
  87. data/lib/codeminer/expressions/until_expression.rb +23 -0
  88. data/lib/codeminer/expressions/variable_expression.rb +20 -0
  89. data/lib/codeminer/expressions/void_expression.rb +21 -0
  90. data/lib/codeminer/expressions/when_expression.rb +24 -0
  91. data/lib/codeminer/expressions/while_expression.rb +23 -0
  92. data/lib/codeminer/expressions/xstring_expression.rb +24 -0
  93. data/lib/codeminer/expressions/yield_expression.rb +22 -0
  94. data/lib/codeminer/parser.rb +215 -0
  95. data/lib/codeminer/processors/argument_processor.rb +25 -0
  96. data/lib/codeminer/processors/array_processor.rb +86 -0
  97. data/lib/codeminer/processors/assignment_processor.rb +42 -0
  98. data/lib/codeminer/processors/binary_processor.rb +12 -0
  99. data/lib/codeminer/processors/call_processor.rb +32 -0
  100. data/lib/codeminer/processors/class_processor.rb +19 -0
  101. data/lib/codeminer/processors/command_processor.rb +15 -0
  102. data/lib/codeminer/processors/condition_processor.rb +59 -0
  103. data/lib/codeminer/processors/default_processor.rb +33 -0
  104. data/lib/codeminer/processors/hash_processor.rb +27 -0
  105. data/lib/codeminer/processors/loop_processor.rb +30 -0
  106. data/lib/codeminer/processors/method_processor.rb +58 -0
  107. data/lib/codeminer/processors/params_processor.rb +97 -0
  108. data/lib/codeminer/processors/range_processor.rb +15 -0
  109. data/lib/codeminer/processors/regexp_processor.rb +27 -0
  110. data/lib/codeminer/processors/return_processor.rb +15 -0
  111. data/lib/codeminer/processors/string_processor.rb +79 -0
  112. data/lib/codeminer/processors/symbol_processor.rb +31 -0
  113. data/lib/codeminer/processors/token_processor.rb +41 -0
  114. data/lib/codeminer/processors/variable_processor.rb +70 -0
  115. data/lib/codeminer/source_extract.rb +120 -0
  116. data/lib/codeminer/substitution.rb +48 -0
  117. data/lib/codeminer/token.rb +19 -0
  118. metadata +172 -0
@@ -0,0 +1,26 @@
1
+ module CodeMiner
2
+
3
+ class SuperExpression < Expression
4
+
5
+ attr_reader :value, :body
6
+
7
+ def initialize(body, src)
8
+ @body = body
9
+ @src = src
10
+ end
11
+
12
+ def type
13
+ :super
14
+ end
15
+
16
+ def each
17
+ if body
18
+ body.each
19
+ else
20
+ []
21
+ end
22
+ end
23
+
24
+ end
25
+
26
+ end
@@ -0,0 +1,32 @@
1
+ module CodeMiner
2
+
3
+ class SymbolExpression < Expression
4
+
5
+ def self.convert(exp)
6
+ if exp.kind_of?(Token)
7
+ new(exp, exp.src_extract)
8
+ else
9
+ exp
10
+ end
11
+ end
12
+
13
+ def initialize(token, src)
14
+ @token = token
15
+ @src = src
16
+ end
17
+
18
+ def type
19
+ :symbol
20
+ end
21
+
22
+ def value
23
+ @token.value
24
+ end
25
+
26
+ def each
27
+ []
28
+ end
29
+
30
+ end
31
+
32
+ end
@@ -0,0 +1,22 @@
1
+ module CodeMiner
2
+
3
+ class TrueExpression < Expression
4
+
5
+ attr_reader :value
6
+
7
+ def initialize(token, src)
8
+ @token = token
9
+ @src = src
10
+ end
11
+
12
+ def type
13
+ :true
14
+ end
15
+
16
+ def each
17
+ []
18
+ end
19
+
20
+ end
21
+
22
+ end
@@ -0,0 +1,23 @@
1
+ module CodeMiner
2
+
3
+ class UnaryExpression < Expression
4
+
5
+ attr_reader :value, :receiver
6
+
7
+ def initialize(value, receiver, src)
8
+ @value = value
9
+ @receiver = receiver
10
+ @src = src
11
+ end
12
+
13
+ def type
14
+ :unary
15
+ end
16
+
17
+ def each
18
+ [@receiver]
19
+ end
20
+
21
+ end
22
+
23
+ end
@@ -0,0 +1,22 @@
1
+ module CodeMiner
2
+
3
+ class UndefExpression < Expression
4
+
5
+ attr_reader :value
6
+
7
+ def initialize(symbols, src)
8
+ @symbols = symbols
9
+ @src = src
10
+ end
11
+
12
+ def type
13
+ :undef
14
+ end
15
+
16
+ def each
17
+ @symbols
18
+ end
19
+
20
+ end
21
+
22
+ end
@@ -0,0 +1,35 @@
1
+ module CodeMiner
2
+
3
+ class UnknownExpression < Expression
4
+
5
+ attr_reader :type, :value, :line, :column
6
+
7
+ def initialize(type, value, src, children, line, column)
8
+ @type, @value, @src, @children, @line, @column = type, value, src, children, line, column
9
+ end
10
+
11
+ def <<(obj)
12
+ @children << obj
13
+ end
14
+
15
+ def each
16
+ @children
17
+ end
18
+
19
+ def add(obj, src)
20
+ @children << obj
21
+ @src = src
22
+ self
23
+ end
24
+
25
+ def end_line
26
+ line
27
+ end
28
+
29
+ def end_column
30
+ column + value.to_s.length
31
+ end
32
+
33
+ end
34
+
35
+ end
@@ -0,0 +1,23 @@
1
+ module CodeMiner
2
+
3
+ class UntilExpression < Expression
4
+
5
+ attr_reader :test, :body, :value
6
+
7
+ def initialize(test, body, src)
8
+ @test = test
9
+ @body = body
10
+ @src = src
11
+ end
12
+
13
+ def type
14
+ :until
15
+ end
16
+
17
+ def each
18
+ [@test, @body]
19
+ end
20
+
21
+ end
22
+
23
+ end
@@ -0,0 +1,20 @@
1
+ module CodeMiner
2
+
3
+ class VariableExpression < Expression
4
+
5
+ def initialize(token, src)
6
+ @token = token
7
+ @src = src
8
+ end
9
+
10
+ def type
11
+ :var_ref
12
+ end
13
+
14
+ def each
15
+ [@token]
16
+ end
17
+
18
+ end
19
+
20
+ end
@@ -0,0 +1,21 @@
1
+ module CodeMiner
2
+
3
+ class VoidExpression < Expression
4
+
5
+ attr_reader :value
6
+
7
+ def initialize(src)
8
+ @src = src
9
+ end
10
+
11
+ def type
12
+ :void_stmt
13
+ end
14
+
15
+ def each
16
+ []
17
+ end
18
+
19
+ end
20
+
21
+ end
@@ -0,0 +1,24 @@
1
+ module CodeMiner
2
+
3
+ class WhenExpression < Expression
4
+
5
+ attr_reader :value, :else, :body
6
+
7
+ def initialize(args, body, else_exp, src)
8
+ @args = args
9
+ @body = body
10
+ @else = else_exp
11
+ @src = src
12
+ end
13
+
14
+ def type
15
+ :when
16
+ end
17
+
18
+ def each
19
+ [@args, @body]
20
+ end
21
+
22
+ end
23
+
24
+ end
@@ -0,0 +1,23 @@
1
+ module CodeMiner
2
+
3
+ class WhileExpression < Expression
4
+
5
+ attr_reader :test, :body, :value
6
+
7
+ def initialize(test, body, src)
8
+ @test = test
9
+ @body = body
10
+ @src = src
11
+ end
12
+
13
+ def type
14
+ :while
15
+ end
16
+
17
+ def each
18
+ [@test, @body]
19
+ end
20
+
21
+ end
22
+
23
+ end
@@ -0,0 +1,24 @@
1
+ module CodeMiner
2
+
3
+ class XStringExpression < Expression
4
+
5
+ def initialize(token, src)
6
+ @token = token
7
+ @src = src
8
+ end
9
+
10
+ def value
11
+ @token.value
12
+ end
13
+
14
+ def type
15
+ :xstring
16
+ end
17
+
18
+ def each
19
+ @token.each
20
+ end
21
+
22
+ end
23
+
24
+ end
@@ -0,0 +1,22 @@
1
+ module CodeMiner
2
+
3
+ class YieldExpression < Expression
4
+
5
+ attr_reader :body, :value
6
+
7
+ def initialize(body, src)
8
+ @body = body
9
+ @src = src
10
+ end
11
+
12
+ def type
13
+ :yield
14
+ end
15
+
16
+ def each
17
+ [@body]
18
+ end
19
+
20
+ end
21
+
22
+ end
@@ -0,0 +1,215 @@
1
+ require 'ripper'
2
+ require File.expand_path('../processors/argument_processor', __FILE__)
3
+ require File.expand_path('../processors/array_processor', __FILE__)
4
+ require File.expand_path('../processors/assignment_processor', __FILE__)
5
+ require File.expand_path('../processors/binary_processor', __FILE__)
6
+ require File.expand_path('../processors/call_processor', __FILE__)
7
+ require File.expand_path('../processors/command_processor', __FILE__)
8
+ require File.expand_path('../processors/class_processor', __FILE__)
9
+ require File.expand_path('../processors/condition_processor', __FILE__)
10
+ require File.expand_path('../processors/default_processor', __FILE__)
11
+ require File.expand_path('../processors/hash_processor', __FILE__)
12
+ require File.expand_path('../processors/loop_processor', __FILE__)
13
+ require File.expand_path('../processors/method_processor', __FILE__)
14
+ require File.expand_path('../processors/params_processor', __FILE__)
15
+ require File.expand_path('../processors/range_processor', __FILE__)
16
+ require File.expand_path('../processors/regexp_processor', __FILE__)
17
+ require File.expand_path('../processors/return_processor', __FILE__)
18
+ require File.expand_path('../processors/string_processor', __FILE__)
19
+ require File.expand_path('../processors/symbol_processor', __FILE__)
20
+ require File.expand_path('../processors/token_processor', __FILE__)
21
+ require File.expand_path('../processors/variable_processor', __FILE__)
22
+
23
+ require File.expand_path('../expression_processor', __FILE__)
24
+ require File.expand_path('../sexp_processor', __FILE__)
25
+
26
+
27
+ module CodeMiner
28
+
29
+ class Parser < Ripper
30
+
31
+ include AssignmentProcessor, CallProcessor, ClassProcessor, MethodProcessor, RegexpProcessor, TokenProcessor,
32
+ StringProcessor, ConditionProcessor, BinaryProcessor, ReturnProcessor, ParamsProcessor, ArgumentProcessor,
33
+ HashProcessor, SymbolProcessor, VariableProcessor, ArrayProcessor, CommandProcessor, LoopProcessor,
34
+ RangeProcessor,
35
+ DefaultProcessor
36
+
37
+ attr_accessor :processors, :formatters
38
+
39
+ def initialize(src, *args)
40
+ @src = src
41
+ @keywords = []
42
+ @begin_regex = []
43
+ @embexpr = []
44
+ @string_begin = []
45
+ @string_end = []
46
+ @symbol_begin = []
47
+ @string_content = []
48
+ @parens_begin = []
49
+ @qwords = []
50
+ @qsymbols = []
51
+ @lambda = []
52
+ @symbols = []
53
+ @words = []
54
+ @processors = []
55
+ @lbrace = []
56
+ @formatters = {}
57
+ super
58
+ end
59
+
60
+ def self.process(rb, processors)
61
+ parser = new(rb).extend(ExpressionProcessor)
62
+ parser.processors = processors
63
+ parser.parse
64
+ processors
65
+ end
66
+
67
+ def self.sexp(rb, formatters={})
68
+ parser = new(rb).extend(SexpProcessor)
69
+ parser.formatters = formatters
70
+ parser.parse
71
+ end
72
+
73
+ def on_break(args)
74
+ BreakExpression.new(args, extract_src_by_token(pop_keyword('break')))
75
+ end
76
+
77
+ def on_next(args)
78
+ NextExpression.new(args, extract_src_by_token(pop_keyword('next')))
79
+ end
80
+
81
+ def on_redo
82
+ RedoExpression.new(extract_src_by_token(pop_keyword('redo')))
83
+ end
84
+
85
+ def on_retry
86
+ RetryExpression.new(extract_src_by_token(pop_keyword('retry')))
87
+ end
88
+
89
+ def on_zsuper
90
+ token = pop_keyword('super')
91
+ SuperExpression.new(nil, extract_src_by_token(token))
92
+ end
93
+
94
+ def on_super(body)
95
+ token = pop_keyword('super')
96
+ SuperExpression.new(body, extract_src_by_tokens(*[token, body].compact))
97
+ end
98
+
99
+ def on_alias(left, right)
100
+ AliasExpression.new(left, right, extract_src_by_token(pop_keyword('alias')))
101
+ end
102
+
103
+ def on_yield(args)
104
+ YieldExpression.new(args, extract_src_by_token(pop_keyword('yield')))
105
+ end
106
+
107
+ def on_yield0
108
+ YieldExpression.new(nil, extract_src_by_token(pop_keyword('yield')))
109
+ end
110
+
111
+ def on_undef(symbols)
112
+ UndefExpression.new(symbols, extract_src_by_token(pop_keyword('undef')))
113
+ end
114
+
115
+ def on_int(*)
116
+ token = super
117
+ IntExpression.new(token, extract_src_by_tokens(token))
118
+ end
119
+
120
+ def on_unary(value, receiver)
121
+ UnaryExpression.new(value.to_s.chomp('@'), receiver, extract_src(receiver.line, receiver.column - value.length - unary_offset(value), receiver.end_line, receiver.end_column))
122
+ end
123
+
124
+ def on_void_stmt
125
+ VoidExpression.new(extract_src(lineno(), column()))
126
+ end
127
+
128
+ def on_begin(body)
129
+ BeginExpression.new(body, extract_src_by_token(pop_keyword('begin')))
130
+ end
131
+
132
+ def on_rescue(constants, assign, body, d)
133
+ constants ||= []
134
+ rescue_body = RescueBodyExpression.wrap(body)
135
+ src = extract_src_by_tokens(pop_keyword('rescue'), rescue_body)
136
+ RescueExpression.new(constants.each.to_a, assign, rescue_body, d, src)
137
+ end
138
+
139
+ def on_ensure(body)
140
+ EnsureExpression.new(body, extract_src_by_tokens(pop_keyword('ensure'), body))
141
+ end
142
+
143
+ def on_bodystmt(body, rescue_exp, else_exp, ensure_exp)
144
+ BodystmtExpression.new(body, rescue_exp, else_exp, ensure_exp, src: extract_src_by_tokens(body, body))
145
+ end
146
+
147
+ def on_stmts_new
148
+ BodyExpression.new(extract_src(lineno(), column()))
149
+ end
150
+
151
+ def on_stmts_add(body, statement)
152
+ assert body.kind_of?(BodyExpression), "Expected body #{body} to be type of BodyExpression"
153
+ body.add(statement)
154
+ end
155
+
156
+ def on_defined(body)
157
+ DefinedExpression.new(body, extract_src_by_tokens(pop_keyword('defined?'),body))
158
+ end
159
+
160
+ def on_program(body)
161
+ RootExpression.new(body, @src)
162
+ end
163
+
164
+ def on_operator_ambiguous(value, msg)
165
+ token = Token.new(:op, value.to_s, extract_src_by_value(value))
166
+ AmbiguousOperatorExpression.new(token, msg, extract_src_by_token(token))
167
+ end
168
+
169
+ def on_magic_comment(key, value)
170
+ MagicCommentExpression.new(key, value, extract_src(lineno(), column()))
171
+ end
172
+
173
+ def compile_error(msg)
174
+ raise CodeMiner::ParseError, msg
175
+ end
176
+
177
+ private
178
+
179
+ def extract_src_by_token(token, line=lineno(), col=column())
180
+ SourceExtract.extract_by_token(@src, token, line, col)
181
+ end
182
+
183
+ def extract_src_by_tokens(begin_token, end_token=begin_token)
184
+ SourceExtract.extract_by_tokens(@src, begin_token, end_token)
185
+ end
186
+
187
+ def extract_src_by_value(value)
188
+ extract_src(lineno(), column(), lineno(), column() + value.length)
189
+ end
190
+
191
+ def extract_src(begin_line, begin_column, end_line=lineno(), end_column=column())
192
+ SourceExtract.new(@src, begin_line, begin_column, end_line, end_column)
193
+ end
194
+
195
+ def extract_params_source(params)
196
+ extract_src(params.first.line, params.last.line, params.first.column, params.last.column)
197
+ end
198
+
199
+ def assert(t, msg)
200
+ raise CodeMiner::ParseError, msg unless t
201
+ end
202
+
203
+ def pop_keyword(type)
204
+ i = @keywords.index {|k| k.value == type }
205
+ assert(i, "Expected to find keyword of type #{type}")
206
+ @keywords.delete_at(i)
207
+ end
208
+
209
+ def unary_offset(value)
210
+ value == :-@ ? -1 : 1
211
+ end
212
+
213
+ end
214
+
215
+ end