crystalizer 0.2.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (126) hide show
  1. data/Changelog +27 -0
  2. data/README +79 -0
  3. data/Rakefile +24 -0
  4. data/TODO +14 -0
  5. data/VERSION +1 -0
  6. data/benchmarks/bench.rb +129 -0
  7. data/benchmarks/concretize_test.rb +26 -0
  8. data/benchmarks/extconf.rb +10 -0
  9. data/benchmarks/tak_rb.rb +7 -0
  10. data/benchmarks/tak_so.rb +7 -0
  11. data/benchmarks/tak_source.rb +16 -0
  12. data/bin/rb2cx +162 -0
  13. data/doc/eval2c.txt +246 -0
  14. data/doc/gen_html.rb +26 -0
  15. data/doc/html_template +10 -0
  16. data/doc/index.txt +169 -0
  17. data/doc/limitations.txt +529 -0
  18. data/doc/optimizations.txt +185 -0
  19. data/doc/rb2cx.txt +130 -0
  20. data/doc/style.css +27 -0
  21. data/lib/concretizer.rb +3 -0
  22. data/lib/ruby2cext/c_function.rb +617 -0
  23. data/lib/ruby2cext/common_node_comp.rb +1412 -0
  24. data/lib/ruby2cext/compiler.rb +311 -0
  25. data/lib/ruby2cext/concretize.rb +269 -0
  26. data/lib/ruby2cext/error.rb +15 -0
  27. data/lib/ruby2cext/eval2c.rb +126 -0
  28. data/lib/ruby2cext/parser.rb +36 -0
  29. data/lib/ruby2cext/plugin.rb +24 -0
  30. data/lib/ruby2cext/plugins/builtin_methods.rb +817 -0
  31. data/lib/ruby2cext/plugins/cache_call.rb +293 -0
  32. data/lib/ruby2cext/plugins/case_optimize.rb +102 -0
  33. data/lib/ruby2cext/plugins/const_cache.rb +36 -0
  34. data/lib/ruby2cext/plugins/direct_self_call.rb +70 -0
  35. data/lib/ruby2cext/plugins/inline_builtin.rb +797 -0
  36. data/lib/ruby2cext/plugins/inline_methods.rb +68 -0
  37. data/lib/ruby2cext/plugins/ivar_cache.rb +147 -0
  38. data/lib/ruby2cext/plugins/require_include.rb +69 -0
  39. data/lib/ruby2cext/plugins/util.rb +154 -0
  40. data/lib/ruby2cext/plugins/warnings.rb +121 -0
  41. data/lib/ruby2cext/scopes.rb +225 -0
  42. data/lib/ruby2cext/str_to_c_strlit.rb +12 -0
  43. data/lib/ruby2cext/tools.rb +80 -0
  44. data/lib/ruby2cext/version.rb +22 -0
  45. data/results +68 -0
  46. data/setup.rb +1585 -0
  47. data/stuff/builtin_methods.rb +69 -0
  48. data/stuff/builtin_methods_test.rb +37 -0
  49. data/test/bootstrap.rb +10 -0
  50. data/test/causes_crash_all_opts.rb +1165 -0
  51. data/test/eval2c/test_eval2c.rb +37 -0
  52. data/test/temp_17.rb +16 -0
  53. data/test/temp_18.rb +8 -0
  54. data/test/temp_19.rb +8 -0
  55. data/test/temp_2.rb +7 -0
  56. data/test/temp_20.rb +5 -0
  57. data/test/temp_21.rb +161 -0
  58. data/test/temp_22.rb +7 -0
  59. data/test/temp_23.rb +7 -0
  60. data/test/temp_24.rb +219 -0
  61. data/test/temp_25.rb +7 -0
  62. data/test/temp_26.rb +11 -0
  63. data/test/temp_27.rb +11 -0
  64. data/test/temp_28.rb +9 -0
  65. data/test/temp_29.rb +9 -0
  66. data/test/temp_3.rb +0 -0
  67. data/test/temp_30.rb +0 -0
  68. data/test/temp_31.rb +10 -0
  69. data/test/temp_32.rb +10 -0
  70. data/test/temp_33.rb +15 -0
  71. data/test/temp_34.rb +15 -0
  72. data/test/temp_35.rb +7 -0
  73. data/test/temp_36.rb +7 -0
  74. data/test/temp_37.rb +10 -0
  75. data/test/temp_38.rb +10 -0
  76. data/test/temp_39.rb +0 -0
  77. data/test/temp_4.rb +7 -0
  78. data/test/temp_40.rb +50 -0
  79. data/test/temp_41.rb +50 -0
  80. data/test/temp_42.rb +8 -0
  81. data/test/temp_43.rb +8 -0
  82. data/test/temp_44.rb +0 -0
  83. data/test/temp_48.rb +7 -0
  84. data/test/temp_49.rb +7 -0
  85. data/test/temp_5.rb +7 -0
  86. data/test/temp_59.rb +7 -0
  87. data/test/temp_6.rb +7 -0
  88. data/test/temp_60.rb +7 -0
  89. data/test/temp_68.rb +239 -0
  90. data/test/temp_7.rb +7 -0
  91. data/test/temp_70.rb +7 -0
  92. data/test/temp_71.rb +7 -0
  93. data/test/temp_72.rb +13 -0
  94. data/test/temp_73.rb +7 -0
  95. data/test/temp_74.rb +7 -0
  96. data/test/temp_76.rb +7 -0
  97. data/test/temp_77.rb +13 -0
  98. data/test/temp_79.rb +7 -0
  99. data/test/temp_8.rb +14 -0
  100. data/test/temp_81.rb +14 -0
  101. data/test/temp_83.rb +0 -0
  102. data/test/temp_84.rb +7 -0
  103. data/test/temp_85.rb +7 -0
  104. data/test/temp_86.rb +14 -0
  105. data/test/temp_87.rb +7 -0
  106. data/test/temp_88.rb +7 -0
  107. data/test/temp_89.rb +7 -0
  108. data/test/temp_9.rb +14 -0
  109. data/test/temp_90.rb +0 -0
  110. data/test/temp_91.rb +7 -0
  111. data/test/temp_92.rb +7 -0
  112. data/test/temp_93.rb +7 -0
  113. data/test/temp_94.rb +7 -0
  114. data/test/temp_95.rb +7 -0
  115. data/test/temp_96.rb +0 -0
  116. data/test/temp_97.rb +0 -0
  117. data/test/temp_98.rb +7 -0
  118. data/test/temp_99.rb +7 -0
  119. data/test/test_concretize.rb +132 -0
  120. data/test/test_concretize_all.rb +15 -0
  121. data/test/test_crystalize_block.rb +73 -0
  122. data/test/test_files/test.rb +615 -0
  123. data/test/test_files/vmode_test.rb +73 -0
  124. data/test/test_files/warn_test.rb +35 -0
  125. data/test/test_syntax.rb +25 -0
  126. metadata +268 -0
@@ -0,0 +1,27 @@
1
+ Changelog
2
+
3
+ -- 0.2.0
4
+
5
+ * New features:
6
+ * New plugin architecture for optimizations and other things
7
+ * Optimization plugins: const_cache, builtin_methods, inline_methods,
8
+ case_optimize
9
+ * Other plugins: require_include, warnings
10
+ * Eval2C for compilation at runtime
11
+ * Available as gem now
12
+
13
+ * Improvements:
14
+ * Implemented control flow "through" rescue/ensure clauses
15
+ * Better vmode handling
16
+ * Support for Ruby 1.8.5 and 1.8.6
17
+ * Improved performance for global variables
18
+
19
+ * Fixes:
20
+ * Workaround for an issue with ALLOCA_N in while loops
21
+ * Fixed a problem with case/when (missing newline nodes)
22
+ * Removed libraries from compile command
23
+ * Various other small fixes
24
+
25
+ -- 0.1.0
26
+
27
+ * Initial release
data/README ADDED
@@ -0,0 +1,79 @@
1
+ ----------- JIT -------------
2
+
3
+ The crystalizer is a loose wrapper for a ruby2c converter called ruby2cext. It takes your ruby code, creates its rubyC equivalent, and re-loadsthe C version over the top of it, thus effectively JIT-ing your existing Ruby code. It is called crystalizer because it effectively should be called *after* setting up existing classes and methods, so that when it can "crystalize" the calls to existing methods, by cacheing their location. Thus, it is not 100% ruby compatible, and currently loses some backtrace information. It has proven quite a bit faster than 1.9 in some benchmarks, though it is currently only 1.8 compatible [presumably if it were 1.9 compatible, it might be even faster].
4
+
5
+ Synopsis:
6
+
7
+ require 'concretizer'
8
+
9
+ klass.concretize! # concretizes all interior methods of this class
10
+
11
+ of, if you have several classes and don't want to have to remember exactly which ones to concretize:
12
+
13
+ Ruby2CExtension::Concretize.crystalize_after_first_time_through {
14
+ # do some stuff here
15
+ # after completing this block, it will concretize any and all classes it ran into in the course of the block
16
+ # so the second and third times through this block should be faster [or any other code that subsequently calls those classes]
17
+ }
18
+
19
+ And for the daring:
20
+
21
+ Ruby2CExtension::Conretize.concretize_all! # optimizes all known classes in the current Ruby VM -- takes forever, might discover bugs in edge cases that aren't covered.
22
+
23
+ Also an option is the rb2cx executable
24
+ $ rb2cx ruby_filename.rb
25
+
26
+ creates ruby_filename.{c, so} based on ruby_filename.rb.
27
+ Thus you could run the ruby script's c equivalent thus:
28
+ $ ruby -rruby_filename.so -e ''
29
+
30
+
31
+ Much thanks to the original author of Ruby2CExtension, since this is only a wrapper to his work.
32
+
33
+ Note that crystalize won't crystalize existing procs in the system--they stay as pure Ruby--it crystalizes methods and future procs created from/within those methods, however.
34
+
35
+ Ruby2CExtension
36
+ ===============
37
+
38
+ Ruby2CExtension is a Ruby to C extension translator/compiler. It takes any Ruby
39
+ source file, parses it using Ruby's builtin parser and then translates the
40
+ abstract syntax tree into "equivalent" C extension code.
41
+
42
+
43
+ Requirements
44
+ ------------
45
+
46
+ * Ruby 1.8.4, 1.8.5 or 1.8.6 (possibly 1.8.7--haven't tried)
47
+ * RubyNode (available at http://rubynode.rubyforge.org/)
48
+
49
+
50
+ Installation
51
+ ------------
52
+
53
+ Just run (as root): TODO
54
+
55
+ gem install ruby2cext
56
+
57
+ Or if you do not use the gem:
58
+
59
+ ruby setup.rb
60
+
61
+ This will install Ruby2CExtension to the default locations. Optionally you can
62
+ supply some options to setup.rb to customize the installation (see "ruby
63
+ setup.rb --help").
64
+
65
+
66
+ Documentation
67
+ -------------
68
+
69
+ Please see doc/index.html for more documentation.
70
+
71
+
72
+ License
73
+ -------
74
+
75
+ Copyright 2006-2007 Dominik Bathon <dbatml@gmx.de>
76
+
77
+ Ruby2CExtension is licensed under the same terms as Ruby.
78
+
79
+ http://github.com/rdp/crystalizer
@@ -0,0 +1,24 @@
1
+ begin
2
+ require 'jeweler'
3
+ Jeweler::Tasks.new do |s|
4
+ s.name = "crystalizer"
5
+ # 0.2.0.
6
+ # 2 actually works with concretize now
7
+ s.author = "Dominik Bathon, rogerdpack"
8
+ s.email = "dbatml@gmx.de"
9
+ # s.homepage = "http://ruby2cext.rubyforge.org/"
10
+ s.summary = "Ruby2CExtension is a Ruby to C extension translator/compiler/concretizer."
11
+ s.add_dependency("rubynode", ">= 0.1.1")
12
+ s.add_dependency("sane")
13
+ s.add_dependency("event_hook")
14
+ s.files.exclude '**/temp*'
15
+ s.add_dependency("backports")
16
+ s.add_dependency("ruby2ruby")
17
+ s.add_dependency("ParseTree")
18
+ s.add_development_dependency("assert2")
19
+ s.add_development_dependency("backtracer")
20
+ end
21
+
22
+ rescue LoadError
23
+ puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
24
+ end
data/TODO ADDED
@@ -0,0 +1,14 @@
1
+ benchmark it against 1.9, etc.
2
+
3
+ make sure it works in linux [should pass tests]
4
+
5
+ can it run rails? do I need to avoid binding/eval methods etc?
6
+
7
+ benchmarks on rails?
8
+
9
+ fix inline_builtin [meh]
10
+ fix the other bwokded on
11
+
12
+ make it compile faster [possibilities: fork, can methods compare to those we know are C-y?]
13
+
14
+ is it faster to compile object's methods down into the class itself? [benchmarks]
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.2.2
@@ -0,0 +1,129 @@
1
+ # some benchmarks from ruby2cext that it used against itself
2
+
3
+ require "benchmark"
4
+
5
+ def fib(n)
6
+ if n < 2
7
+ n
8
+ else
9
+ fib(n - 1) + fib(n - 2)
10
+ end
11
+ end
12
+
13
+ def mkmatrix(rows, cols)
14
+ count = 1
15
+ mx = Array.new(rows)
16
+ (0 .. (rows - 1)).each do |bi|
17
+ row = Array.new(cols, 0)
18
+ (0 .. (cols - 1)).each do |j|
19
+ row[j] = count
20
+ count += 1
21
+ end
22
+ mx[bi] = row
23
+ end
24
+ mx
25
+ end
26
+
27
+ def mmult(rows, cols, m1, m2)
28
+ m3 = Array.new(rows)
29
+ (0 .. (rows - 1)).each do |bi|
30
+ row = Array.new(cols, 0)
31
+ (0 .. (cols - 1)).each do |j|
32
+ val = 0
33
+ (0 .. (cols - 1)).each do |k|
34
+ val += m1.at(bi).at(k) * m2.at(k).at(j)
35
+ end
36
+ row[j] = val
37
+ end
38
+ m3[bi] = row
39
+ end
40
+ m3
41
+ end
42
+
43
+ def sqrt(x)
44
+ x = x.to_f
45
+ average = lambda { |a, b| (a+b)/2 }
46
+ impr = lambda { |g| average[g, x/g] }
47
+ good_en = lambda { |g| (g*g - x).abs < 0.001 }
48
+ try = lambda { |g| good_en[g] ? g : try[impr[g]] }
49
+ try[1.0]
50
+ end
51
+
52
+
53
+ Benchmark.bm(12) { |bmx|
54
+ bmx.report("times") { 3000000.times{|e| e + e } }
55
+ bmx.report("times") { 3000000.times{|e| e + e } }
56
+ bmx.report("fib") { fib 30 }
57
+ bmx.report("array") {
58
+ n = 2000
59
+ x = Array.new(n)
60
+ y = Array.new(n, 0)
61
+ n.times{|bi| x[bi] = bi + 1 }
62
+ (0 .. 999).each do |e|
63
+ (n-1).step(0,-1) do |bi|
64
+ y[bi] += x.at(bi)
65
+ end
66
+ end
67
+ }
68
+ bmx.report("matrix") {
69
+ n = 40
70
+ size = 30
71
+ m1 = mkmatrix(size, size)
72
+ m2 = mkmatrix(size, size)
73
+ mm = Array.new
74
+ n.times do
75
+ mm = mmult(size, size, m1, m2)
76
+ end
77
+ }
78
+ bmx.report("while") {
79
+ i = 3000000
80
+ while i > 0
81
+ break if i == 5
82
+ i -= 1
83
+ next if i % 100 == 0
84
+ redo if (i-=1) % 100 == 1
85
+ end
86
+ }
87
+ bmx.report("sqrt") {
88
+ i = 40000
89
+ while (i-=1) > 0
90
+ sqrt 2
91
+ end
92
+ }
93
+ bmx.report("3 [] alloc") {
94
+ i = 1000000
95
+ while (i-=1) > 0
96
+ [[], []]
97
+ end
98
+ }
99
+ bmx.report("const cache") {
100
+ 1000000.times {
101
+ a = Array
102
+ a = Object::Array
103
+ a = ::Array
104
+ }
105
+ }
106
+ bmx.report("method calls") {
107
+ max = 2000
108
+ z = x = 0
109
+ while (x+=1) <= max
110
+ y = 0
111
+ while (y+=1) <= max
112
+ z = (x+y-z) % 32000
113
+ end
114
+ end
115
+ }
116
+ bmx.report("ivar") {
117
+ Class.new {
118
+ def run
119
+ @i = 4000000
120
+ while 0 < (@i = -1 + @i)
121
+ a = @i
122
+ b = @i
123
+ end
124
+ end
125
+ }.new.run
126
+ }
127
+ }
128
+
129
+
@@ -0,0 +1,26 @@
1
+ require 'rubygems'
2
+ require 'benchmark'
3
+ require '../lib/concretizer'
4
+
5
+
6
+ at_exit {
7
+ if false #$!
8
+ puts "==== "
9
+ puts $!.backtrace.join("\n")
10
+ puts "===="
11
+ end
12
+ }
13
+
14
+
15
+ class A
16
+ def go
17
+ end
18
+ def go2
19
+ go
20
+ end
21
+ end
22
+
23
+ a = A.new
24
+ 3.times {puts Benchmark.realtime { 4000000.times { a.go }} }
25
+ puts Benchmark.realtime { Ruby2CExtension::Concretize.concretize_all! }
26
+ 3.times { puts Benchmark.realtime { 4000000.times { a.go }} }
@@ -0,0 +1,10 @@
1
+ require 'mkmf'
2
+
3
+ # Give it a name
4
+ extension_name = 'tak_source'
5
+
6
+ # The destination
7
+ dir_config(extension_name)
8
+
9
+ # Do the work
10
+ create_makefile(extension_name)
@@ -0,0 +1,7 @@
1
+ require './tak_source.rb'
2
+
3
+ require 'benchmark'
4
+
5
+ [7, 8, 9].each do |n|
6
+ print n, " took ", Benchmark.realtime { tak(18, n, 0) }, "\n"
7
+ end
@@ -0,0 +1,7 @@
1
+ require './tak_source.so'
2
+
3
+ require 'benchmark'
4
+
5
+ [7, 8, 9].each do |n|
6
+ print n, " took ", Benchmark.realtime { tak(18, n, 0) }, "\n"
7
+ end
@@ -0,0 +1,16 @@
1
+ public
2
+ def tak(x, y, z)
3
+ unless (y < x)
4
+ z
5
+ else
6
+ tak(tak(x-1, y, z),
7
+ tak(y-1, z, x),
8
+ tak(z-1, x, y))
9
+ end
10
+ end
11
+
12
+ require 'benchmark'
13
+
14
+ [7, 8, 9].each do |n|
15
+ print n, " took ", Benchmark.realtime { tak(18, n, 0) }, "\n"
16
+ end
@@ -0,0 +1,162 @@
1
+ #!/usr/bin/env ruby
2
+ if $0 == __FILE__
3
+ require 'rubygems'
4
+ $:.unshift '../lib' # in case you use it during development
5
+ end
6
+
7
+ require "ruby2cext/compiler"
8
+ require "ruby2cext/version"
9
+ require "getoptlong"
10
+ require "logger"
11
+
12
+ include Ruby2CExtension
13
+
14
+ def usage(logger)
15
+ logger.warn(<<EOS.strip)
16
+ Usage: rb2cx [options] file.rb ...
17
+
18
+ Translates the given Ruby file into an equivalent C extension. The result is
19
+ stored in file.c. It will then be compiled into a shared object file, unless
20
+ the option --only-c is given.
21
+
22
+ If multiple files are given, each file will be handled separately.
23
+
24
+ === General Options:
25
+
26
+ -h / --help print this help
27
+ -c / --only-c only translate to C
28
+ -v / --verbose print status messages
29
+ -w / --warnings print warnings for things that might not work as expected
30
+ -V / --version print the Ruby2CExtension version
31
+
32
+ === Include Option:
33
+
34
+ -I / --include path
35
+
36
+ If a Ruby file "require"s another Ruby file and that file can be found in the
37
+ given path, then it will be included in the C extension. This option can be
38
+ used multiple times with different paths, the paths are then searched in the
39
+ given order. Use --verbose to see which files were included.
40
+
41
+ === Optimization Options:
42
+
43
+ -O / --optimization <optimization>
44
+
45
+ Where <optimization> is one of the following:
46
+
47
+ const_cache
48
+ enables local constant lookup caching
49
+
50
+ builtin_methods
51
+ optimizes calls to many methods of builtin types
52
+
53
+ inline_methods
54
+ inlines the methods nil?, equal? and __send__
55
+
56
+ case_optimize
57
+ optimizes case statments with nil, true, false or Fixnums
58
+
59
+ inline_builtin
60
+ inlines methods calls to builtin types
61
+
62
+ cache_call
63
+ caches method lookup for each call
64
+
65
+ direct_self_call
66
+ directly calls private or recursive methods defined in same scope
67
+
68
+ all
69
+ enables all of the above optimizations
70
+
71
+ === Examples:
72
+
73
+ rb2cx -wv file.rb
74
+ rb2cx -I . -O all file.rb
75
+ rb2cx -I . -I ../libs -O const_cache -O builtin_methods -w file.rb
76
+ EOS
77
+ end
78
+
79
+ def main
80
+ opts = GetoptLong.new(
81
+ ["--help", "-h", GetoptLong::NO_ARGUMENT],
82
+ ["--only-c", "-c", GetoptLong::NO_ARGUMENT],
83
+ ["--verbose", "-v", GetoptLong::NO_ARGUMENT],
84
+ ["--version", "-V", GetoptLong::NO_ARGUMENT],
85
+ ["--debug", GetoptLong::NO_ARGUMENT], # undocumented
86
+ ["--warnings", "-w", GetoptLong::NO_ARGUMENT],
87
+ ["--include", "-I", GetoptLong::REQUIRED_ARGUMENT],
88
+ ["--optimization", "-O", GetoptLong::REQUIRED_ARGUMENT]
89
+ )
90
+
91
+ logger = Logger.new(STDERR)
92
+ logger.formatter = proc { |severity, time, progname, msg| "#{msg}\n" }
93
+ logger.level = Logger::INFO
94
+
95
+ only_c = false
96
+ include_paths = []
97
+ optimizations = {}
98
+ all_optimizations = false
99
+ plugins = {}
100
+ version_printed = false
101
+ begin
102
+ opts.each do |opt, arg|
103
+ case opt
104
+ when "--help"
105
+ usage(logger)
106
+ exit
107
+ when "--only-c"
108
+ only_c = true
109
+ when "--verbose"
110
+ logger.level = Logger::INFO
111
+ when "--version"
112
+ unless version_printed
113
+ logger.warn(Ruby2CExtension::FULL_VERSION_STRING)
114
+ version_printed = true
115
+ end
116
+ when "--debug"
117
+ logger.level = Logger::DEBUG
118
+ when "--warnings"
119
+ plugins[:warnings] = true
120
+ when "--include"
121
+ unless File.directory?(arg)
122
+ raise "'#{arg}' is no directory"
123
+ end
124
+ include_paths << arg
125
+ when "--optimization"
126
+ case (arg = arg.to_sym)
127
+ when :const_cache,
128
+ :case_optimize,
129
+ :inline_builtin,
130
+ :inline_methods,
131
+ :cache_call,
132
+ :builtin_methods,
133
+ :direct_self_call
134
+ optimizations[arg] = true
135
+ when :all
136
+ all_optimizations = true
137
+ else
138
+ raise "unknown optimization: #{arg}"
139
+ end
140
+ end
141
+ end
142
+ if ARGV.empty? && !version_printed
143
+ raise "No files given"
144
+ end
145
+ rescue => e
146
+ logger.error("#{e} ('rb2cx --help' for help)")
147
+ exit 1
148
+ end
149
+
150
+ plugins[:optimizations] = all_optimizations ? :all : optimizations
151
+
152
+ begin
153
+ ARGV.each { |fn|
154
+ Compiler.compile_file(fn, plugins, include_paths, only_c, logger)
155
+ }
156
+ rescue RuntimeError, SyntaxError => e
157
+ logger.error(e)
158
+ exit 1
159
+ end
160
+ end
161
+
162
+ main