live_ast 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. data/CHANGES.rdoc +6 -0
  2. data/MANIFEST +54 -0
  3. data/README.rdoc +388 -0
  4. data/Rakefile +19 -0
  5. data/devel/jumpstart.rb +983 -0
  6. data/lib/live_ast/ast_eval.rb +13 -0
  7. data/lib/live_ast/ast_load.rb +15 -0
  8. data/lib/live_ast/base.rb +56 -0
  9. data/lib/live_ast/cache.rb +14 -0
  10. data/lib/live_ast/error.rb +30 -0
  11. data/lib/live_ast/evaler.rb +66 -0
  12. data/lib/live_ast/linker.rb +107 -0
  13. data/lib/live_ast/loader.rb +69 -0
  14. data/lib/live_ast/parser.rb +48 -0
  15. data/lib/live_ast/replace_load.rb +14 -0
  16. data/lib/live_ast/replace_raise.rb +21 -0
  17. data/lib/live_ast/to_ast.rb +17 -0
  18. data/lib/live_ast/to_ruby.rb +12 -0
  19. data/lib/live_ast/version.rb +3 -0
  20. data/lib/live_ast.rb +4 -0
  21. data/test/ast_eval_feature_test.rb +11 -0
  22. data/test/ast_load_feature_test.rb +11 -0
  23. data/test/backtrace_test.rb +159 -0
  24. data/test/covert_define_method_test.rb +23 -0
  25. data/test/def_test.rb +35 -0
  26. data/test/define_method_test.rb +41 -0
  27. data/test/define_singleton_method_test.rb +15 -0
  28. data/test/encoding_test/bad.rb +1 -0
  29. data/test/encoding_test/cp932.rb +6 -0
  30. data/test/encoding_test/default.rb +5 -0
  31. data/test/encoding_test/eucjp.rb +6 -0
  32. data/test/encoding_test/koi8.rb +6 -0
  33. data/test/encoding_test/koi8_shebang.rb +7 -0
  34. data/test/encoding_test/usascii.rb +6 -0
  35. data/test/encoding_test/utf8.rb +6 -0
  36. data/test/encoding_test.rb +51 -0
  37. data/test/error_test.rb +115 -0
  38. data/test/eval_test.rb +269 -0
  39. data/test/flush_cache_test.rb +98 -0
  40. data/test/lambda_test.rb +56 -0
  41. data/test/load_path_test.rb +84 -0
  42. data/test/load_test.rb +85 -0
  43. data/test/noninvasive_test.rb +51 -0
  44. data/test/readme_test.rb +11 -0
  45. data/test/recursive_eval_test.rb +52 -0
  46. data/test/redefine_method_test.rb +83 -0
  47. data/test/reload_test.rb +108 -0
  48. data/test/shared/ast_generators.rb +124 -0
  49. data/test/shared/main.rb +110 -0
  50. data/test/stdlib_test.rb +11 -0
  51. data/test/thread_test.rb +44 -0
  52. data/test/to_ast_feature_test.rb +15 -0
  53. data/test/to_ruby_feature_test.rb +15 -0
  54. data/test/to_ruby_test.rb +86 -0
  55. metadata +223 -0
data/CHANGES.rdoc ADDED
@@ -0,0 +1,6 @@
1
+
2
+ = LiveAST ChangeLog
3
+
4
+ == Version 0.2.0
5
+
6
+ * initial release
data/MANIFEST ADDED
@@ -0,0 +1,54 @@
1
+ CHANGES.rdoc
2
+ MANIFEST
3
+ README.rdoc
4
+ Rakefile
5
+ devel/jumpstart.rb
6
+ lib/live_ast.rb
7
+ lib/live_ast/ast_eval.rb
8
+ lib/live_ast/ast_load.rb
9
+ lib/live_ast/base.rb
10
+ lib/live_ast/cache.rb
11
+ lib/live_ast/error.rb
12
+ lib/live_ast/evaler.rb
13
+ lib/live_ast/linker.rb
14
+ lib/live_ast/loader.rb
15
+ lib/live_ast/parser.rb
16
+ lib/live_ast/replace_load.rb
17
+ lib/live_ast/replace_raise.rb
18
+ lib/live_ast/to_ast.rb
19
+ lib/live_ast/to_ruby.rb
20
+ lib/live_ast/version.rb
21
+ test/ast_eval_feature_test.rb
22
+ test/ast_load_feature_test.rb
23
+ test/backtrace_test.rb
24
+ test/covert_define_method_test.rb
25
+ test/def_test.rb
26
+ test/define_method_test.rb
27
+ test/define_singleton_method_test.rb
28
+ test/encoding_test.rb
29
+ test/encoding_test/bad.rb
30
+ test/encoding_test/cp932.rb
31
+ test/encoding_test/default.rb
32
+ test/encoding_test/eucjp.rb
33
+ test/encoding_test/koi8.rb
34
+ test/encoding_test/koi8_shebang.rb
35
+ test/encoding_test/usascii.rb
36
+ test/encoding_test/utf8.rb
37
+ test/error_test.rb
38
+ test/eval_test.rb
39
+ test/flush_cache_test.rb
40
+ test/lambda_test.rb
41
+ test/load_path_test.rb
42
+ test/load_test.rb
43
+ test/noninvasive_test.rb
44
+ test/readme_test.rb
45
+ test/recursive_eval_test.rb
46
+ test/redefine_method_test.rb
47
+ test/reload_test.rb
48
+ test/shared/ast_generators.rb
49
+ test/shared/main.rb
50
+ test/stdlib_test.rb
51
+ test/thread_test.rb
52
+ test/to_ast_feature_test.rb
53
+ test/to_ruby_feature_test.rb
54
+ test/to_ruby_test.rb
data/README.rdoc ADDED
@@ -0,0 +1,388 @@
1
+
2
+ = LiveAST
3
+
4
+ == Summary
5
+
6
+ A pure Ruby library for obtaining live abstract syntax trees of
7
+ methods and procs.
8
+
9
+ == Synopsis
10
+
11
+ require 'live_ast'
12
+
13
+ class Greet
14
+ def default
15
+ "hello"
16
+ end
17
+ end
18
+
19
+ #### ASTs of methods
20
+
21
+ m = Greet.instance_method(:default)
22
+
23
+ p m.to_ast
24
+ # => s(:defn, :default, s(:args), s(:scope, s(:block, s(:str, "hello"))))
25
+
26
+ #### ASTs of lambdas, procs, blocks
27
+
28
+ f = lambda { "foo" }
29
+
30
+ p f.to_ast
31
+ # => s(:iter, s(:call, nil, :lambda, s(:arglist)), nil, s(:str, "foo"))
32
+
33
+ def query(&block)
34
+ p block.to_ast
35
+ # => s(:iter, s(:call, nil, :query, s(:arglist)), nil, s(:str, "bar"))
36
+ end
37
+
38
+ query do
39
+ "bar"
40
+ end
41
+
42
+ #### ASTs from dynamic code
43
+
44
+ f = ast_eval "lambda { 'dynamic' }", binding
45
+
46
+ p f.to_ast
47
+ # => s(:iter, s(:call, nil, :lambda, s(:arglist)), nil, s(:str, "dynamic"))
48
+
49
+ ast_eval "def g ; 'dynamic' ; end", binding
50
+ m = method(:g)
51
+
52
+ p m.to_ast
53
+ # => s(:defn, :g, s(:args), s(:scope, s(:block, s(:str, "dynamic"))))
54
+
55
+ == Install
56
+
57
+ % gem install live_ast
58
+
59
+ == Description
60
+
61
+ LiveAST enables a program to find the ASTs of objects created by
62
+ dynamically generated code. It may be used in a strictly noninvasive
63
+ manner, where no standard classes or methods are modified, or it may
64
+ be transparently integrated into Ruby (experimental). The default
65
+ setting is in between.
66
+
67
+ RubyParser is responsible for parsing and building the ASTs, though
68
+ another parser may be easily substituted (in fact the name +to_ast+ is
69
+ used instead of +to_sexp+ because LiveAST has no understanding of what
70
+ the parser outputs).
71
+
72
+ LiveAST is thread-safe.
73
+
74
+ Ruby 1.9.2 or higher is required.
75
+
76
+ == Links
77
+
78
+ * Documentation: http://liveast.rubyforge.org
79
+ * Rubyforge home: http://rubyforge.org/projects/liveast/
80
+ * Repository: http://github.com/quix/live_ast
81
+
82
+ == +to_ruby+
83
+
84
+ Although the ruby2ruby package (<code>gem install ruby2ruby</code>) is
85
+ not an official dependency of LiveAST, for convenience
86
+
87
+ require 'live_ast/to_ruby'
88
+
89
+ will require ruby2ruby and define the +to_ruby+ method for +Method+,
90
+ +UnboundMethod+, and +Proc+. These methods are one-liners which pass
91
+ the extracted ASTs to ruby2ruby.
92
+
93
+ require 'live_ast'
94
+ require 'live_ast/to_ruby'
95
+
96
+ p lambda { |x, y| x + y }.to_ruby # => "lambda { |x, y| (x + y) }"
97
+
98
+ class A
99
+ def f
100
+ "A#f"
101
+ end
102
+ end
103
+
104
+ p A.instance_method(:f).to_ruby # => "def f\n \"A#f\"\nend"
105
+
106
+ == Loading Source
107
+
108
+ Objects created via +require+ and +load+ will be AST-accessible.
109
+ However +ast_eval+ must be used instead of +eval+ for AST-accessible
110
+ objects. +ast_eval+ has the same semantics as +eval+ except that the
111
+ binding argument is required.
112
+
113
+ require 'live_ast'
114
+
115
+ class A
116
+ ast_eval %{
117
+ def f
118
+ "A#f"
119
+ end
120
+
121
+ # nested evals OK
122
+ ast_eval %{
123
+ def g
124
+ "A#g"
125
+ end
126
+ }, binding
127
+
128
+ }, binding
129
+ end
130
+
131
+ p A.instance_method(:f).to_ast
132
+ # => s(:defn, :f, s(:args), s(:scope, s(:block, s(:str, "A#f"))))
133
+
134
+ p A.instance_method(:g).to_ast
135
+ # => s(:defn, :g, s(:args), s(:scope, s(:block, s(:str, "A#g"))))
136
+
137
+ == Limitations
138
+
139
+ A method or block definition must not share a line with other methods
140
+ or blocks in order for its AST to be available.
141
+
142
+ require 'live_ast'
143
+
144
+ class A
145
+ def f ; end ; def g ; end
146
+ end
147
+ A.instance_method(:f).to_ast # => raises LiveAST::MultipleDefinitionsOnSameLineError
148
+
149
+ a = lambda { } ; b = lambda { }
150
+ a.to_ast # => raises LiveAST::MultipleDefinitionsOnSameLineError
151
+
152
+ == Technical Issues
153
+
154
+ You can probably ignore this section. Goodbye.
155
+
156
+ === Replacing the Parser
157
+
158
+ Despite its name, LiveAST knows nothing about ASTs. It merely reports
159
+ what it finds in the line-to-AST hash obtained from
160
+ <code>LiveAST::Parser#parse</code>. Replacing the +Parser+ class is
161
+ therefore easy: the only specification is that the +parse+ method
162
+ return such a hash.
163
+
164
+ Supposing your new <code>LiveAST::Parser</code> is defined in
165
+ <code>/path/to/my/code/live_ast/parser.rb</code>,
166
+
167
+ $LOAD_PATH.unshift '/path/to/my/code'
168
+ require 'live_ast'
169
+
170
+ will override LiveAST's default parser with your own. To test it,
171
+ provide some examples of what your ASTs look like in
172
+ <code>tests/shared/ast_generators.rb</code>.
173
+
174
+ === Noninvasive Mode
175
+
176
+ For safety purposes, <code>require 'live_ast'</code> performs the
177
+ invasive act of redefining +load+ (but not +require+); otherwise bad
178
+ things can happen to the unwary. The addition of +to_ast+ to a few
179
+ standard Ruby classes is also a meddlesome move.
180
+
181
+ To avoid these modifications,
182
+
183
+ require 'live_ast/base'
184
+
185
+ will provide the essentials of LiveAST but will not touch core classes
186
+ or methods.
187
+
188
+ To select features individually,
189
+
190
+ require 'live_ast/to_ast' # define to_ast for Method, UnboundMethod, Proc
191
+ require 'live_ast/to_ruby' # define to_ruby for Method, UnboundMethod, Proc
192
+ require 'live_ast/ast_eval' # define Kernel#ast_eval
193
+ require 'live_ast/ast_load' # define Kernel#ast_load (mentioned below)
194
+ require 'live_ast/replace_load' # redefine Kernel#load
195
+
196
+ === Noninvasive Interface
197
+
198
+ The following alternative interface is available.
199
+
200
+ require 'live_ast/base'
201
+
202
+ class A
203
+ def f
204
+ "A#f"
205
+ end
206
+ end
207
+
208
+ p LiveAST.ast(A.instance_method(:f))
209
+ # => s(:defn, :f, s(:args), s(:scope, s(:block, s(:str, "A#f"))))
210
+
211
+ p LiveAST.ast(lambda { })
212
+ # => s(:iter, s(:call, nil, :lambda, s(:arglist)), nil, nil)
213
+
214
+ f = LiveAST.eval("lambda { }", binding)
215
+
216
+ p LiveAST.ast(f)
217
+ # => s(:iter, s(:call, nil, :lambda, s(:arglist)), nil, nil)
218
+
219
+ ast_eval # => raises NameError
220
+
221
+ === Reloading Files In Noninvasive Mode
222
+
223
+ Use +ast_load+ or (equivalently) <code>LiveAST.load</code> when
224
+ reloading an AST-aware file.
225
+
226
+ require 'live_ast/ast_load'
227
+ require 'live_ast/to_ast'
228
+
229
+ require "foo"
230
+ Foo.instance_method(:bar).to_ast # caches AST
231
+
232
+ # ... the bar method is changed in foo.rb ...
233
+
234
+ ast_load "foo.rb"
235
+ p Foo.instance_method(:bar).to_ast # => updated AST
236
+
237
+ Note if +load+ is called instead of +ast_load+ then the last line will
238
+ give the old AST,
239
+
240
+ load "foo.rb" # oops! forgot to use ast_load
241
+ p Foo.instance_method(:bar).to_ast # => stale AST
242
+
243
+ Realize that +foo.rb+ may be referenced by an unknown number of
244
+ methods and blocks. If the original +foo.rb+ source were dumped in
245
+ favor of the modified +foo.rb+, then an unknown number of those
246
+ references would be invalidated (and some may even point to the wrong
247
+ AST!).
248
+
249
+ This is the reason for the caching that results in the stale AST
250
+ above. It should now be clear why the default behavior of
251
+ <code>require 'live_ast'</code> is to redefine +load+: doing so
252
+ prevents this problem entirely. On the other hand if it is fully known
253
+ where and when files are being reloaded (if at all) then there's no
254
+ need for paranoia; this noninvasive option may be most the
255
+ appropriate.
256
+
257
+ === The Source/AST Cache
258
+
259
+ +ast_eval+ and +load+ cache all incoming code, while
260
+ <code>require</code>d files are cached on a need-to-know basis. When
261
+ an AST is requested, the corresponding source is parsed and discarded,
262
+ leaving behind method and block ASTs. +to_ast+ removes an AST from the
263
+ cache and attaches it to the appropriate object (a Proc or Module).
264
+
265
+ Ignored, unextracted ASTs will therefore linger in the cache. Since
266
+ sexps are generally small there is little need for concern unless one
267
+ is continually evaling/reloading <em>and</em> failing to extract the
268
+ sexps. Nevertheless it is possible that old ASTs will eventually need
269
+ to be garbage collected. To flush the cache,
270
+
271
+ (1) Check that +to_ast+ has been called on all objects whose ASTs are
272
+ desired.
273
+
274
+ (2) Call <code>LiveAST.flush_cache</code>.
275
+
276
+ Calling +to_ast+ prevents the object's AST from being flushed (since
277
+ it grafts the AST onto the object).
278
+
279
+ ASTs of procs and methods whose sources lie in <code>require</code>d
280
+ files will never be flushed. However a method redefined via +ast_eval+
281
+ or +load+ is susceptible to +flush_cache+ even when its original
282
+ definition pointed to a <code>require</code>d file.
283
+
284
+ === About +require+
285
+
286
+ No measures have been taken to detect manipulations of
287
+ <code>$LOADED_FEATURES</code> which would cause +require+ to load the
288
+ same file twice. Though +require+ <em>could</em> be replaced in
289
+ similar fashion to +load+--heading off problems arising from such
290
+ "raw" reloads--the overhead would seem inappropriate in relation to
291
+ the rarity of this case.
292
+
293
+ Therefore the working assumption is that +require+ will load a file
294
+ only once. Furthermore, if a file has not been reloaded then it is
295
+ assumed that the file is unmodified between the moment it is
296
+ <code>require</code>d and the moment the first AST is pulled from it.
297
+
298
+ === Backtraces
299
+
300
+ +ast_eval+ is meant to be compatible with +eval+. For instance the
301
+ first line of the backtrace should be identical to that of +eval+:
302
+
303
+ require 'live_ast'
304
+
305
+ ast_eval %{ raise "boom" }, binding
306
+ # => test.rb:3:in `<main>': boom (RuntimeError)
307
+
308
+ Let's make a slight change,
309
+
310
+ require 'live_ast'
311
+
312
+ f = ast_eval %{ lambda { raise "boom" } }, binding
313
+ f.call
314
+ # => test.rb|ast@a:3:in `block in <main>': boom (RuntimeError)
315
+
316
+ Oh no! What the heck is '<code>|ast@a</code>' doing there? LiveAST's
317
+ implementation has just been exposed: each source input is assigned a
318
+ unique key that enables a Ruby object to find its own definition.
319
+
320
+ In the first case above, +ast_eval+ has removed the key from the
321
+ exception backtrace. But in the second case there is no opportunity to
322
+ remove it since +ast_eval+ has already returned.
323
+
324
+ If you find this to be problem--for example if you cannot add a filter
325
+ for the jump-to-location feature in your editor--then +raise+ may be
326
+ redefined to strip these tokens,
327
+
328
+ require 'live_ast'
329
+ require 'live_ast/replace_raise'
330
+
331
+ f = ast_eval %{ lambda { raise "boom" } }, binding
332
+ f.call
333
+ # => test.rb:4:in `block in <main>': boom (RuntimeError)
334
+
335
+ However this only applies to a +raise+ call originating from Ruby
336
+ code. An exception raised within a native method will likely still
337
+ contain the token in its backtrace (e.g., in MRI the exception raised
338
+ by <code>1/0</code> comes from C). In principle this could be fixed by
339
+ having the Ruby interpreter dynamically call +raise+.
340
+
341
+
342
+ === Replacing +eval+
343
+
344
+ If +ast_eval+ did not require a binding argument then it could assume
345
+ the role of +eval+, thereby making LiveAST fully transparent to the
346
+ user. Is this possible in pure Ruby?
347
+
348
+ The only option which has been investigated thus far is MRI, which can
349
+ summon <code>Binding.of_caller</code> (recently rewritten for 1.9.2)
350
+ to fill in the missing binding argument. Unfortunately a limitation
351
+ with tracing events in MRI places a few odd restrictions on the syntax
352
+ surrounding +eval+, though all restrictions can be trivially
353
+ sidestepped. Nonetheless it does work (it passes rubyspec minus the
354
+ backtrace issue) despite being somewhat impractical.
355
+
356
+ This (mis)feature is maintained in a separate branch named
357
+ +replace_eval+ on github (not part of the gem). For more information see
358
+ replace_eval.rb[http://github.com/quix/live_ast/blob/replace_eval/lib/live_ast/replace_eval.rb].
359
+
360
+ == Author
361
+
362
+ * James M. Lawrence < quixoticsycophant@gmail.com >
363
+
364
+ == License
365
+
366
+ Copyright (c) 2011 James M. Lawrence. All rights reserved.
367
+
368
+ Permission is hereby granted, free of charge, to any person
369
+ obtaining a copy of this software and associated documentation files
370
+ (the "Software"), to deal in the Software without restriction,
371
+ including without limitation the rights to use, copy, modify, merge,
372
+ publish, distribute, sublicense, and/or sell copies of the Software,
373
+ and to permit persons to whom the Software is furnished to do so,
374
+ subject to the following conditions:
375
+
376
+ The above copyright notice and this permission notice shall be
377
+ included in all copies or substantial portions of the Software.
378
+
379
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
380
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
381
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
382
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
383
+ BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
384
+ ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
385
+ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
386
+ SOFTWARE.
387
+
388
+
data/Rakefile ADDED
@@ -0,0 +1,19 @@
1
+ require_relative 'devel/jumpstart'
2
+
3
+ Jumpstart.new "live_ast" do |s|
4
+ s.developer "James M. Lawrence", "quixoticsycophant@gmail.com"
5
+ s.rubyforge_user = "quix"
6
+ s.rubyforge_name = "liveast"
7
+ s.camel_name = "LiveAST"
8
+ s.rdoc_title = "LiveAST: Live Abstract Syntax Trees"
9
+
10
+ # my code compensates for a ruby_parser bug; make this equal for now
11
+ s.dependency("ruby_parser", "= 2.0.5")
12
+
13
+ s.rdoc_files = %w[
14
+ README.rdoc
15
+ lib/live_ast/ast_eval.rb
16
+ lib/live_ast/base.rb
17
+ lib/live_ast/version.rb
18
+ ]
19
+ end