crystalizer 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. data/Changelog +27 -0
  2. data/README +79 -0
  3. data/Rakefile +24 -0
  4. data/TODO +14 -0
  5. data/VERSION +1 -0
  6. data/benchmarks/bench.rb +129 -0
  7. data/benchmarks/concretize_test.rb +26 -0
  8. data/benchmarks/extconf.rb +10 -0
  9. data/benchmarks/tak_rb.rb +7 -0
  10. data/benchmarks/tak_so.rb +7 -0
  11. data/benchmarks/tak_source.rb +16 -0
  12. data/bin/rb2cx +162 -0
  13. data/doc/eval2c.txt +246 -0
  14. data/doc/gen_html.rb +26 -0
  15. data/doc/html_template +10 -0
  16. data/doc/index.txt +169 -0
  17. data/doc/limitations.txt +529 -0
  18. data/doc/optimizations.txt +185 -0
  19. data/doc/rb2cx.txt +130 -0
  20. data/doc/style.css +27 -0
  21. data/lib/concretizer.rb +3 -0
  22. data/lib/ruby2cext/c_function.rb +617 -0
  23. data/lib/ruby2cext/common_node_comp.rb +1412 -0
  24. data/lib/ruby2cext/compiler.rb +311 -0
  25. data/lib/ruby2cext/concretize.rb +269 -0
  26. data/lib/ruby2cext/error.rb +15 -0
  27. data/lib/ruby2cext/eval2c.rb +126 -0
  28. data/lib/ruby2cext/parser.rb +36 -0
  29. data/lib/ruby2cext/plugin.rb +24 -0
  30. data/lib/ruby2cext/plugins/builtin_methods.rb +817 -0
  31. data/lib/ruby2cext/plugins/cache_call.rb +293 -0
  32. data/lib/ruby2cext/plugins/case_optimize.rb +102 -0
  33. data/lib/ruby2cext/plugins/const_cache.rb +36 -0
  34. data/lib/ruby2cext/plugins/direct_self_call.rb +70 -0
  35. data/lib/ruby2cext/plugins/inline_builtin.rb +797 -0
  36. data/lib/ruby2cext/plugins/inline_methods.rb +68 -0
  37. data/lib/ruby2cext/plugins/ivar_cache.rb +147 -0
  38. data/lib/ruby2cext/plugins/require_include.rb +69 -0
  39. data/lib/ruby2cext/plugins/util.rb +154 -0
  40. data/lib/ruby2cext/plugins/warnings.rb +121 -0
  41. data/lib/ruby2cext/scopes.rb +225 -0
  42. data/lib/ruby2cext/str_to_c_strlit.rb +12 -0
  43. data/lib/ruby2cext/tools.rb +80 -0
  44. data/lib/ruby2cext/version.rb +22 -0
  45. data/results +68 -0
  46. data/setup.rb +1585 -0
  47. data/stuff/builtin_methods.rb +69 -0
  48. data/stuff/builtin_methods_test.rb +37 -0
  49. data/test/bootstrap.rb +10 -0
  50. data/test/causes_crash_all_opts.rb +1165 -0
  51. data/test/eval2c/test_eval2c.rb +37 -0
  52. data/test/temp_17.rb +16 -0
  53. data/test/temp_18.rb +8 -0
  54. data/test/temp_19.rb +8 -0
  55. data/test/temp_2.rb +7 -0
  56. data/test/temp_20.rb +5 -0
  57. data/test/temp_21.rb +161 -0
  58. data/test/temp_22.rb +7 -0
  59. data/test/temp_23.rb +7 -0
  60. data/test/temp_24.rb +219 -0
  61. data/test/temp_25.rb +7 -0
  62. data/test/temp_26.rb +11 -0
  63. data/test/temp_27.rb +11 -0
  64. data/test/temp_28.rb +9 -0
  65. data/test/temp_29.rb +9 -0
  66. data/test/temp_3.rb +0 -0
  67. data/test/temp_30.rb +0 -0
  68. data/test/temp_31.rb +10 -0
  69. data/test/temp_32.rb +10 -0
  70. data/test/temp_33.rb +15 -0
  71. data/test/temp_34.rb +15 -0
  72. data/test/temp_35.rb +7 -0
  73. data/test/temp_36.rb +7 -0
  74. data/test/temp_37.rb +10 -0
  75. data/test/temp_38.rb +10 -0
  76. data/test/temp_39.rb +0 -0
  77. data/test/temp_4.rb +7 -0
  78. data/test/temp_40.rb +50 -0
  79. data/test/temp_41.rb +50 -0
  80. data/test/temp_42.rb +8 -0
  81. data/test/temp_43.rb +8 -0
  82. data/test/temp_44.rb +0 -0
  83. data/test/temp_48.rb +7 -0
  84. data/test/temp_49.rb +7 -0
  85. data/test/temp_5.rb +7 -0
  86. data/test/temp_59.rb +7 -0
  87. data/test/temp_6.rb +7 -0
  88. data/test/temp_60.rb +7 -0
  89. data/test/temp_68.rb +239 -0
  90. data/test/temp_7.rb +7 -0
  91. data/test/temp_70.rb +7 -0
  92. data/test/temp_71.rb +7 -0
  93. data/test/temp_72.rb +13 -0
  94. data/test/temp_73.rb +7 -0
  95. data/test/temp_74.rb +7 -0
  96. data/test/temp_76.rb +7 -0
  97. data/test/temp_77.rb +13 -0
  98. data/test/temp_79.rb +7 -0
  99. data/test/temp_8.rb +14 -0
  100. data/test/temp_81.rb +14 -0
  101. data/test/temp_83.rb +0 -0
  102. data/test/temp_84.rb +7 -0
  103. data/test/temp_85.rb +7 -0
  104. data/test/temp_86.rb +14 -0
  105. data/test/temp_87.rb +7 -0
  106. data/test/temp_88.rb +7 -0
  107. data/test/temp_89.rb +7 -0
  108. data/test/temp_9.rb +14 -0
  109. data/test/temp_90.rb +0 -0
  110. data/test/temp_91.rb +7 -0
  111. data/test/temp_92.rb +7 -0
  112. data/test/temp_93.rb +7 -0
  113. data/test/temp_94.rb +7 -0
  114. data/test/temp_95.rb +7 -0
  115. data/test/temp_96.rb +0 -0
  116. data/test/temp_97.rb +0 -0
  117. data/test/temp_98.rb +7 -0
  118. data/test/temp_99.rb +7 -0
  119. data/test/test_concretize.rb +132 -0
  120. data/test/test_concretize_all.rb +15 -0
  121. data/test/test_crystalize_block.rb +73 -0
  122. data/test/test_files/test.rb +615 -0
  123. data/test/test_files/vmode_test.rb +73 -0
  124. data/test/test_files/warn_test.rb +35 -0
  125. data/test/test_syntax.rb +25 -0
  126. metadata +268 -0
@@ -0,0 +1,27 @@
1
+ Changelog
2
+
3
+ -- 0.2.0
4
+
5
+ * New features:
6
+ * New plugin architecture for optimizations and other things
7
+ * Optimization plugins: const_cache, builtin_methods, inline_methods,
8
+ case_optimize
9
+ * Other plugins: require_include, warnings
10
+ * Eval2C for compilation at runtime
11
+ * Available as gem now
12
+
13
+ * Improvements:
14
+ * Implemented control flow "through" rescue/ensure clauses
15
+ * Better vmode handling
16
+ * Support for Ruby 1.8.5 and 1.8.6
17
+ * Improved performance for global variables
18
+
19
+ * Fixes:
20
+ * Workaround for an issue with ALLOCA_N in while loops
21
+ * Fixed a problem with case/when (missing newline nodes)
22
+ * Removed libraries from compile command
23
+ * Various other small fixes
24
+
25
+ -- 0.1.0
26
+
27
+ * Initial release
data/README ADDED
@@ -0,0 +1,79 @@
1
+ ----------- JIT -------------
2
+
3
+ The crystalizer is a loose wrapper for a ruby2c converter called ruby2cext. It takes your ruby code, creates its rubyC equivalent, and re-loadsthe C version over the top of it, thus effectively JIT-ing your existing Ruby code. It is called crystalizer because it effectively should be called *after* setting up existing classes and methods, so that when it can "crystalize" the calls to existing methods, by cacheing their location. Thus, it is not 100% ruby compatible, and currently loses some backtrace information. It has proven quite a bit faster than 1.9 in some benchmarks, though it is currently only 1.8 compatible [presumably if it were 1.9 compatible, it might be even faster].
4
+
5
+ Synopsis:
6
+
7
+ require 'concretizer'
8
+
9
+ klass.concretize! # concretizes all interior methods of this class
10
+
11
+ of, if you have several classes and don't want to have to remember exactly which ones to concretize:
12
+
13
+ Ruby2CExtension::Concretize.crystalize_after_first_time_through {
14
+ # do some stuff here
15
+ # after completing this block, it will concretize any and all classes it ran into in the course of the block
16
+ # so the second and third times through this block should be faster [or any other code that subsequently calls those classes]
17
+ }
18
+
19
+ And for the daring:
20
+
21
+ Ruby2CExtension::Conretize.concretize_all! # optimizes all known classes in the current Ruby VM -- takes forever, might discover bugs in edge cases that aren't covered.
22
+
23
+ Also an option is the rb2cx executable
24
+ $ rb2cx ruby_filename.rb
25
+
26
+ creates ruby_filename.{c, so} based on ruby_filename.rb.
27
+ Thus you could run the ruby script's c equivalent thus:
28
+ $ ruby -rruby_filename.so -e ''
29
+
30
+
31
+ Much thanks to the original author of Ruby2CExtension, since this is only a wrapper to his work.
32
+
33
+ Note that crystalize won't crystalize existing procs in the system--they stay as pure Ruby--it crystalizes methods and future procs created from/within those methods, however.
34
+
35
+ Ruby2CExtension
36
+ ===============
37
+
38
+ Ruby2CExtension is a Ruby to C extension translator/compiler. It takes any Ruby
39
+ source file, parses it using Ruby's builtin parser and then translates the
40
+ abstract syntax tree into "equivalent" C extension code.
41
+
42
+
43
+ Requirements
44
+ ------------
45
+
46
+ * Ruby 1.8.4, 1.8.5 or 1.8.6 (possibly 1.8.7--haven't tried)
47
+ * RubyNode (available at http://rubynode.rubyforge.org/)
48
+
49
+
50
+ Installation
51
+ ------------
52
+
53
+ Just run (as root): TODO
54
+
55
+ gem install ruby2cext
56
+
57
+ Or if you do not use the gem:
58
+
59
+ ruby setup.rb
60
+
61
+ This will install Ruby2CExtension to the default locations. Optionally you can
62
+ supply some options to setup.rb to customize the installation (see "ruby
63
+ setup.rb --help").
64
+
65
+
66
+ Documentation
67
+ -------------
68
+
69
+ Please see doc/index.html for more documentation.
70
+
71
+
72
+ License
73
+ -------
74
+
75
+ Copyright 2006-2007 Dominik Bathon <dbatml@gmx.de>
76
+
77
+ Ruby2CExtension is licensed under the same terms as Ruby.
78
+
79
+ http://github.com/rdp/crystalizer
@@ -0,0 +1,24 @@
1
+ begin
2
+ require 'jeweler'
3
+ Jeweler::Tasks.new do |s|
4
+ s.name = "crystalizer"
5
+ # 0.2.0.
6
+ # 2 actually works with concretize now
7
+ s.author = "Dominik Bathon, rogerdpack"
8
+ s.email = "dbatml@gmx.de"
9
+ # s.homepage = "http://ruby2cext.rubyforge.org/"
10
+ s.summary = "Ruby2CExtension is a Ruby to C extension translator/compiler/concretizer."
11
+ s.add_dependency("rubynode", ">= 0.1.1")
12
+ s.add_dependency("sane")
13
+ s.add_dependency("event_hook")
14
+ s.files.exclude '**/temp*'
15
+ s.add_dependency("backports")
16
+ s.add_dependency("ruby2ruby")
17
+ s.add_dependency("ParseTree")
18
+ s.add_development_dependency("assert2")
19
+ s.add_development_dependency("backtracer")
20
+ end
21
+
22
+ rescue LoadError
23
+ puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
24
+ end
data/TODO ADDED
@@ -0,0 +1,14 @@
1
+ benchmark it against 1.9, etc.
2
+
3
+ make sure it works in linux [should pass tests]
4
+
5
+ can it run rails? do I need to avoid binding/eval methods etc?
6
+
7
+ benchmarks on rails?
8
+
9
+ fix inline_builtin [meh]
10
+ fix the other bwokded on
11
+
12
+ make it compile faster [possibilities: fork, can methods compare to those we know are C-y?]
13
+
14
+ is it faster to compile object's methods down into the class itself? [benchmarks]
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.2.2
@@ -0,0 +1,129 @@
1
+ # some benchmarks from ruby2cext that it used against itself
2
+
3
+ require "benchmark"
4
+
5
+ def fib(n)
6
+ if n < 2
7
+ n
8
+ else
9
+ fib(n - 1) + fib(n - 2)
10
+ end
11
+ end
12
+
13
+ def mkmatrix(rows, cols)
14
+ count = 1
15
+ mx = Array.new(rows)
16
+ (0 .. (rows - 1)).each do |bi|
17
+ row = Array.new(cols, 0)
18
+ (0 .. (cols - 1)).each do |j|
19
+ row[j] = count
20
+ count += 1
21
+ end
22
+ mx[bi] = row
23
+ end
24
+ mx
25
+ end
26
+
27
+ def mmult(rows, cols, m1, m2)
28
+ m3 = Array.new(rows)
29
+ (0 .. (rows - 1)).each do |bi|
30
+ row = Array.new(cols, 0)
31
+ (0 .. (cols - 1)).each do |j|
32
+ val = 0
33
+ (0 .. (cols - 1)).each do |k|
34
+ val += m1.at(bi).at(k) * m2.at(k).at(j)
35
+ end
36
+ row[j] = val
37
+ end
38
+ m3[bi] = row
39
+ end
40
+ m3
41
+ end
42
+
43
+ def sqrt(x)
44
+ x = x.to_f
45
+ average = lambda { |a, b| (a+b)/2 }
46
+ impr = lambda { |g| average[g, x/g] }
47
+ good_en = lambda { |g| (g*g - x).abs < 0.001 }
48
+ try = lambda { |g| good_en[g] ? g : try[impr[g]] }
49
+ try[1.0]
50
+ end
51
+
52
+
53
+ Benchmark.bm(12) { |bmx|
54
+ bmx.report("times") { 3000000.times{|e| e + e } }
55
+ bmx.report("times") { 3000000.times{|e| e + e } }
56
+ bmx.report("fib") { fib 30 }
57
+ bmx.report("array") {
58
+ n = 2000
59
+ x = Array.new(n)
60
+ y = Array.new(n, 0)
61
+ n.times{|bi| x[bi] = bi + 1 }
62
+ (0 .. 999).each do |e|
63
+ (n-1).step(0,-1) do |bi|
64
+ y[bi] += x.at(bi)
65
+ end
66
+ end
67
+ }
68
+ bmx.report("matrix") {
69
+ n = 40
70
+ size = 30
71
+ m1 = mkmatrix(size, size)
72
+ m2 = mkmatrix(size, size)
73
+ mm = Array.new
74
+ n.times do
75
+ mm = mmult(size, size, m1, m2)
76
+ end
77
+ }
78
+ bmx.report("while") {
79
+ i = 3000000
80
+ while i > 0
81
+ break if i == 5
82
+ i -= 1
83
+ next if i % 100 == 0
84
+ redo if (i-=1) % 100 == 1
85
+ end
86
+ }
87
+ bmx.report("sqrt") {
88
+ i = 40000
89
+ while (i-=1) > 0
90
+ sqrt 2
91
+ end
92
+ }
93
+ bmx.report("3 [] alloc") {
94
+ i = 1000000
95
+ while (i-=1) > 0
96
+ [[], []]
97
+ end
98
+ }
99
+ bmx.report("const cache") {
100
+ 1000000.times {
101
+ a = Array
102
+ a = Object::Array
103
+ a = ::Array
104
+ }
105
+ }
106
+ bmx.report("method calls") {
107
+ max = 2000
108
+ z = x = 0
109
+ while (x+=1) <= max
110
+ y = 0
111
+ while (y+=1) <= max
112
+ z = (x+y-z) % 32000
113
+ end
114
+ end
115
+ }
116
+ bmx.report("ivar") {
117
+ Class.new {
118
+ def run
119
+ @i = 4000000
120
+ while 0 < (@i = -1 + @i)
121
+ a = @i
122
+ b = @i
123
+ end
124
+ end
125
+ }.new.run
126
+ }
127
+ }
128
+
129
+
@@ -0,0 +1,26 @@
1
+ require 'rubygems'
2
+ require 'benchmark'
3
+ require '../lib/concretizer'
4
+
5
+
6
+ at_exit {
7
+ if false #$!
8
+ puts "==== "
9
+ puts $!.backtrace.join("\n")
10
+ puts "===="
11
+ end
12
+ }
13
+
14
+
15
+ class A
16
+ def go
17
+ end
18
+ def go2
19
+ go
20
+ end
21
+ end
22
+
23
+ a = A.new
24
+ 3.times {puts Benchmark.realtime { 4000000.times { a.go }} }
25
+ puts Benchmark.realtime { Ruby2CExtension::Concretize.concretize_all! }
26
+ 3.times { puts Benchmark.realtime { 4000000.times { a.go }} }
@@ -0,0 +1,10 @@
1
+ require 'mkmf'
2
+
3
+ # Give it a name
4
+ extension_name = 'tak_source'
5
+
6
+ # The destination
7
+ dir_config(extension_name)
8
+
9
+ # Do the work
10
+ create_makefile(extension_name)
@@ -0,0 +1,7 @@
1
+ require './tak_source.rb'
2
+
3
+ require 'benchmark'
4
+
5
+ [7, 8, 9].each do |n|
6
+ print n, " took ", Benchmark.realtime { tak(18, n, 0) }, "\n"
7
+ end
@@ -0,0 +1,7 @@
1
+ require './tak_source.so'
2
+
3
+ require 'benchmark'
4
+
5
+ [7, 8, 9].each do |n|
6
+ print n, " took ", Benchmark.realtime { tak(18, n, 0) }, "\n"
7
+ end
@@ -0,0 +1,16 @@
1
+ public
2
+ def tak(x, y, z)
3
+ unless (y < x)
4
+ z
5
+ else
6
+ tak(tak(x-1, y, z),
7
+ tak(y-1, z, x),
8
+ tak(z-1, x, y))
9
+ end
10
+ end
11
+
12
+ require 'benchmark'
13
+
14
+ [7, 8, 9].each do |n|
15
+ print n, " took ", Benchmark.realtime { tak(18, n, 0) }, "\n"
16
+ end
@@ -0,0 +1,162 @@
1
+ #!/usr/bin/env ruby
2
+ if $0 == __FILE__
3
+ require 'rubygems'
4
+ $:.unshift '../lib' # in case you use it during development
5
+ end
6
+
7
+ require "ruby2cext/compiler"
8
+ require "ruby2cext/version"
9
+ require "getoptlong"
10
+ require "logger"
11
+
12
+ include Ruby2CExtension
13
+
14
+ def usage(logger)
15
+ logger.warn(<<EOS.strip)
16
+ Usage: rb2cx [options] file.rb ...
17
+
18
+ Translates the given Ruby file into an equivalent C extension. The result is
19
+ stored in file.c. It will then be compiled into a shared object file, unless
20
+ the option --only-c is given.
21
+
22
+ If multiple files are given, each file will be handled separately.
23
+
24
+ === General Options:
25
+
26
+ -h / --help print this help
27
+ -c / --only-c only translate to C
28
+ -v / --verbose print status messages
29
+ -w / --warnings print warnings for things that might not work as expected
30
+ -V / --version print the Ruby2CExtension version
31
+
32
+ === Include Option:
33
+
34
+ -I / --include path
35
+
36
+ If a Ruby file "require"s another Ruby file and that file can be found in the
37
+ given path, then it will be included in the C extension. This option can be
38
+ used multiple times with different paths, the paths are then searched in the
39
+ given order. Use --verbose to see which files were included.
40
+
41
+ === Optimization Options:
42
+
43
+ -O / --optimization <optimization>
44
+
45
+ Where <optimization> is one of the following:
46
+
47
+ const_cache
48
+ enables local constant lookup caching
49
+
50
+ builtin_methods
51
+ optimizes calls to many methods of builtin types
52
+
53
+ inline_methods
54
+ inlines the methods nil?, equal? and __send__
55
+
56
+ case_optimize
57
+ optimizes case statments with nil, true, false or Fixnums
58
+
59
+ inline_builtin
60
+ inlines methods calls to builtin types
61
+
62
+ cache_call
63
+ caches method lookup for each call
64
+
65
+ direct_self_call
66
+ directly calls private or recursive methods defined in same scope
67
+
68
+ all
69
+ enables all of the above optimizations
70
+
71
+ === Examples:
72
+
73
+ rb2cx -wv file.rb
74
+ rb2cx -I . -O all file.rb
75
+ rb2cx -I . -I ../libs -O const_cache -O builtin_methods -w file.rb
76
+ EOS
77
+ end
78
+
79
+ def main
80
+ opts = GetoptLong.new(
81
+ ["--help", "-h", GetoptLong::NO_ARGUMENT],
82
+ ["--only-c", "-c", GetoptLong::NO_ARGUMENT],
83
+ ["--verbose", "-v", GetoptLong::NO_ARGUMENT],
84
+ ["--version", "-V", GetoptLong::NO_ARGUMENT],
85
+ ["--debug", GetoptLong::NO_ARGUMENT], # undocumented
86
+ ["--warnings", "-w", GetoptLong::NO_ARGUMENT],
87
+ ["--include", "-I", GetoptLong::REQUIRED_ARGUMENT],
88
+ ["--optimization", "-O", GetoptLong::REQUIRED_ARGUMENT]
89
+ )
90
+
91
+ logger = Logger.new(STDERR)
92
+ logger.formatter = proc { |severity, time, progname, msg| "#{msg}\n" }
93
+ logger.level = Logger::INFO
94
+
95
+ only_c = false
96
+ include_paths = []
97
+ optimizations = {}
98
+ all_optimizations = false
99
+ plugins = {}
100
+ version_printed = false
101
+ begin
102
+ opts.each do |opt, arg|
103
+ case opt
104
+ when "--help"
105
+ usage(logger)
106
+ exit
107
+ when "--only-c"
108
+ only_c = true
109
+ when "--verbose"
110
+ logger.level = Logger::INFO
111
+ when "--version"
112
+ unless version_printed
113
+ logger.warn(Ruby2CExtension::FULL_VERSION_STRING)
114
+ version_printed = true
115
+ end
116
+ when "--debug"
117
+ logger.level = Logger::DEBUG
118
+ when "--warnings"
119
+ plugins[:warnings] = true
120
+ when "--include"
121
+ unless File.directory?(arg)
122
+ raise "'#{arg}' is no directory"
123
+ end
124
+ include_paths << arg
125
+ when "--optimization"
126
+ case (arg = arg.to_sym)
127
+ when :const_cache,
128
+ :case_optimize,
129
+ :inline_builtin,
130
+ :inline_methods,
131
+ :cache_call,
132
+ :builtin_methods,
133
+ :direct_self_call
134
+ optimizations[arg] = true
135
+ when :all
136
+ all_optimizations = true
137
+ else
138
+ raise "unknown optimization: #{arg}"
139
+ end
140
+ end
141
+ end
142
+ if ARGV.empty? && !version_printed
143
+ raise "No files given"
144
+ end
145
+ rescue => e
146
+ logger.error("#{e} ('rb2cx --help' for help)")
147
+ exit 1
148
+ end
149
+
150
+ plugins[:optimizations] = all_optimizations ? :all : optimizations
151
+
152
+ begin
153
+ ARGV.each { |fn|
154
+ Compiler.compile_file(fn, plugins, include_paths, only_c, logger)
155
+ }
156
+ rescue RuntimeError, SyntaxError => e
157
+ logger.error(e)
158
+ exit 1
159
+ end
160
+ end
161
+
162
+ main