crystalruby 0.1.11 → 0.1.13
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +17 -0
- data/README.md +83 -19
- data/examples/adder/adder.rb +10 -0
- data/exe/crystalruby +2 -0
- data/lib/crystalruby/adapter.rb +105 -0
- data/lib/crystalruby/compilation.rb +97 -0
- data/lib/crystalruby/config.rb +58 -8
- data/lib/crystalruby/reactor.rb +184 -0
- data/lib/crystalruby/template.rb +6 -5
- data/lib/crystalruby/templates/function.cr +36 -3
- data/lib/crystalruby/templates/index.cr +87 -12
- data/lib/crystalruby/typemaps.rb +8 -3
- data/lib/crystalruby/version.rb +1 -1
- data/lib/crystalruby.rb +77 -167
- data/lib/module.rb +1 -1
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 49ff0856e24d9a62e6dda014115652431074d4a45e1cb4270c92beeae2baa7ea
|
4
|
+
data.tar.gz: adaa86c18d254bf07704997050133db592dd60dda5ca9aed9f4c2bc34edfd7a9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 599c397e9710bb7dfe528e3ec49b64cfbf61f0d73089dc3476d85587317c894367b09f8b47e062d5cc98646575512421f28860aadeceac62f05588a80830726b
|
7
|
+
data.tar.gz: 60b9896f92cf41389933c80bb21e5c4e899e54648575ca778dcea2ba881876ef3f2539bdc46ab1c22b3f234f1237019454f8726957f5858c83ba59a2065e406f
|
data/Dockerfile
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
FROM ruby:3.3
|
2
|
+
|
3
|
+
MAINTAINER Wouter Coppieters <wc@pico.net.nz>
|
4
|
+
|
5
|
+
RUN apt-get update && apt-get install -y curl gnupg2 software-properties-common lsb-release
|
6
|
+
RUN curl -fsSL https://crystal-lang.org/install.sh | bash
|
7
|
+
WORKDIR /usr/src/app
|
8
|
+
|
9
|
+
COPY Gemfile Gemfile.lock ./
|
10
|
+
COPY crystalruby.gemspec ./
|
11
|
+
COPY lib/crystalruby/version.rb ./lib/crystalruby/version.rb
|
12
|
+
|
13
|
+
RUN bundle install
|
14
|
+
COPY . .
|
15
|
+
|
16
|
+
# Define the command to run your application
|
17
|
+
CMD ["bundle", "exec", "irb"]
|
data/README.md
CHANGED
@@ -43,33 +43,49 @@ E.g.
|
|
43
43
|
require 'crystalruby'
|
44
44
|
require 'benchmark'
|
45
45
|
|
46
|
-
module
|
46
|
+
module PrimeCounter
|
47
47
|
crystalize [n: :int32] => :int32
|
48
|
-
def
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
48
|
+
def count_primes_upto_cr(n)
|
49
|
+
primes = 0
|
50
|
+
(2..n).each do |i|
|
51
|
+
is_prime = true
|
52
|
+
(2..Math.sqrt(i).to_i).each do |j|
|
53
|
+
if i % j == 0
|
54
|
+
is_prime = false
|
55
|
+
break
|
56
|
+
end
|
57
|
+
end
|
58
|
+
primes += 1
|
59
|
+
end
|
60
|
+
primes
|
53
61
|
end
|
54
62
|
|
55
63
|
module_function
|
56
64
|
|
57
|
-
def
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
65
|
+
def count_primes_upto_rb(n)
|
66
|
+
primes = 0
|
67
|
+
(2..n).each do |i|
|
68
|
+
is_prime = true
|
69
|
+
(2..Math.sqrt(i).to_i).each do |j|
|
70
|
+
if i % j == 0
|
71
|
+
is_prime = false
|
72
|
+
break
|
73
|
+
end
|
74
|
+
end
|
75
|
+
primes += 1
|
76
|
+
end
|
77
|
+
primes
|
62
78
|
end
|
63
79
|
end
|
64
80
|
|
65
|
-
|
66
|
-
puts(Benchmark.realtime {
|
67
|
-
|
81
|
+
include PrimeCounter
|
82
|
+
puts(Benchmark.realtime { count_primes_upto_rb(1000_000) })
|
83
|
+
puts(Benchmark.realtime { count_primes_upto_cr(1000_000) })
|
68
84
|
```
|
69
85
|
|
70
86
|
```bash
|
71
|
-
|
72
|
-
0.
|
87
|
+
2.8195170001126826 # Ruby
|
88
|
+
0.3402599999681115 # Crystal
|
73
89
|
```
|
74
90
|
|
75
91
|
_Note_: The first run of the Crystal code will be slower, as it needs to compile the code first. The subsequent runs will be much faster.
|
@@ -121,10 +137,10 @@ end
|
|
121
137
|
### Crystal Compatible
|
122
138
|
|
123
139
|
Some Crystal syntax is not valid Ruby, for methods of this form, we need to
|
124
|
-
define our functions using a :
|
140
|
+
define our functions using a raw: true option
|
125
141
|
|
126
142
|
```ruby
|
127
|
-
crystalize
|
143
|
+
crystalize [a: :int, b: :int] => :int, raw: true
|
128
144
|
def add(a, b)
|
129
145
|
<<~CRYSTAL
|
130
146
|
c = 0_u64
|
@@ -453,6 +469,51 @@ CrystalRuby.compile!
|
|
453
469
|
|
454
470
|
Then you can run this file as part of your build step, to ensure all Crystal code is compiled ahead of time.
|
455
471
|
|
472
|
+
## Concurrency
|
473
|
+
|
474
|
+
While Ruby programs allow multi-threading, Crystal by default uses only a single thread, while utilising Fiber based cooperative-multitasking to allow for concurrent execution. This means that by default, Crystal libraries can not safely be invoked in parallel across multiple Ruby threads.
|
475
|
+
|
476
|
+
To safely expose this behaviour, `crystalruby` implements a Reactor, which multiplexes all Ruby calls to Crystal across a single thread. This way you can safely use `crystalruby` in a multi-threaded Ruby environment.
|
477
|
+
|
478
|
+
By default `crystalruby` methods are blocking/synchronous, this means that for blocking operations, a single crystalruby call can block the entire reactor.
|
479
|
+
|
480
|
+
To allow you to benefit from Crystal's fiber based concurrency, you can use use the `async` option on crystalized ruby methods. This allows several Ruby threads to invoke Crystal code simultaneously.
|
481
|
+
|
482
|
+
E.g.
|
483
|
+
|
484
|
+
```ruby
|
485
|
+
module Sleeper
|
486
|
+
crystalize [] => :void
|
487
|
+
def sleep_sync
|
488
|
+
sleep 2
|
489
|
+
end
|
490
|
+
|
491
|
+
crystalize [] => :void, async: true
|
492
|
+
def sleep_async
|
493
|
+
sleep 2
|
494
|
+
end
|
495
|
+
end
|
496
|
+
```
|
497
|
+
|
498
|
+
```ruby
|
499
|
+
5.times.map{ Thread.new{ Sleeper.sleep_sync } }.each(&:join) # Will take 10 seconds
|
500
|
+
5.times.map{ Thread.new{ Sleeper.sleep_async } }.each(&:join) # Will take 2 seconds (the sleeps are processed concurrently)
|
501
|
+
```
|
502
|
+
|
503
|
+
### Reactor performance
|
504
|
+
|
505
|
+
There is a small amount of overhead to multiplexing calls across a single thread. Ad-hoc testing amounts this to be around 10 nanoseconds per call.
|
506
|
+
For most use-cases this overhead is negligible, especially if the bulk of your CPU heavy task occurs exclusively in Crystal code. However, if you are invoking very fast Crystal code from Ruby in a tight loop (e.g. a simple 1 + 2)
|
507
|
+
then the overhead of the reactor can become significant.
|
508
|
+
|
509
|
+
In this case you can use the `crystalruby` in a single-threaded mode to avoid the reactor overhead and greatly increase performance, with the caveat that _all_ calls to Crystal must occur from a single thread. If you Ruby program is already single-threaded this is not a problem.
|
510
|
+
|
511
|
+
```ruby
|
512
|
+
CrystalRuby.configure do |config|
|
513
|
+
config.single_thread_mode = true
|
514
|
+
end
|
515
|
+
```
|
516
|
+
|
456
517
|
## Troubleshooting
|
457
518
|
|
458
519
|
The logic to detect when to JIT recompile is not robust and can end up in an inconsistent state. To remedy this it is useful to clear out all generated assets and build from scratch.
|
@@ -517,7 +578,7 @@ crystal_codegen_dir: "generated"
|
|
517
578
|
debug: true
|
518
579
|
```
|
519
580
|
|
520
|
-
Alternatively, these can be set programmatically:
|
581
|
+
Alternatively, these can be set programmatically, e.g:
|
521
582
|
|
522
583
|
```ruby
|
523
584
|
CrystalRuby.configure do |config|
|
@@ -527,6 +588,9 @@ CrystalRuby.configure do |config|
|
|
527
588
|
config.crystal_lib_name = "crlib"
|
528
589
|
config.crystal_codegen_dir = "generated"
|
529
590
|
config.debug = true
|
591
|
+
config.verbose = false
|
592
|
+
config.colorize_log_output = false
|
593
|
+
config.log_level = :info
|
530
594
|
end
|
531
595
|
```
|
532
596
|
|
data/exe/crystalruby
CHANGED
@@ -0,0 +1,105 @@
|
|
1
|
+
module CrystalRuby
|
2
|
+
module Adapter
|
3
|
+
# Define a method to set the @crystalize proc if it doesn't already exist
|
4
|
+
def crystalize(raw: false, async: false, **options, &block)
|
5
|
+
(args,), returns = options.first
|
6
|
+
args ||= {}
|
7
|
+
raise "Arguments should be of the form name: :type. Got #{args}" unless args.is_a?(Hash)
|
8
|
+
|
9
|
+
@crystalize_next = { raw: raw, async: async, args: args, returns: returns, block: block }
|
10
|
+
end
|
11
|
+
|
12
|
+
def crystal(raw: false, &block)
|
13
|
+
inline_crystal_body = Template::InlineChunk.render(
|
14
|
+
{
|
15
|
+
module_name: name,
|
16
|
+
body: block.source.lines[
|
17
|
+
raw ? 2...-2 : 1...-1
|
18
|
+
].join("\n")
|
19
|
+
}
|
20
|
+
)
|
21
|
+
CrystalRuby.write_chunk(self, body: inline_crystal_body)
|
22
|
+
end
|
23
|
+
|
24
|
+
def crtype(&block)
|
25
|
+
TypeBuilder.with_injected_type_dsl(self) do
|
26
|
+
TypeBuilder.build(&block)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def json(&block)
|
31
|
+
crtype(&block).serialize_as(:json)
|
32
|
+
end
|
33
|
+
|
34
|
+
def method_added(method_name)
|
35
|
+
if @crystalize_next
|
36
|
+
define_crystalized_method(method_name, instance_method(method_name))
|
37
|
+
@crystalize_next = nil
|
38
|
+
end
|
39
|
+
super
|
40
|
+
end
|
41
|
+
|
42
|
+
def singleton_method_added(method_name)
|
43
|
+
if @crystalize_next
|
44
|
+
define_crystalized_method(method_name, singleton_method(method_name))
|
45
|
+
@crystalize_next = nil
|
46
|
+
end
|
47
|
+
super
|
48
|
+
end
|
49
|
+
|
50
|
+
def define_crystalized_method(method_name, method)
|
51
|
+
CrystalRuby.log_debug("Defining crystalized method #{name}.#{method_name}")
|
52
|
+
CrystalRuby.instantiate_crystal_ruby! unless CrystalRuby.instantiated?
|
53
|
+
|
54
|
+
function_body = method.source.lines[
|
55
|
+
@crystalize_next[:raw] ? 2...-2 : 1...-1
|
56
|
+
].join("\n")
|
57
|
+
|
58
|
+
MethodSource.instance_variable_get(:@lines_for_file).delete(method.source_location[0])
|
59
|
+
lib_fname = "#{name.downcase}_#{method_name}_#{Digest::MD5.hexdigest(function_body)}"
|
60
|
+
args, returns, block, async = @crystalize_next.values_at(:args, :returns, :block, :async)
|
61
|
+
args ||= {}
|
62
|
+
@crystalize_next = nil
|
63
|
+
function = CrystalRuby.build_function(self, lib_fname, method_name, args, returns, function_body)
|
64
|
+
CrystalRuby.write_chunk(self, name: function[:name], body: function[:body]) do
|
65
|
+
CrystalRuby.log_debug("attaching #{lib_fname} to #{name}")
|
66
|
+
extend FFI::Library
|
67
|
+
ffi_lib CrystalRuby.config.crystal_lib_dir / CrystalRuby.config.crystal_lib_name
|
68
|
+
if async
|
69
|
+
attach_function lib_fname, "#{lib_fname}_async", function[:ffi_types] + %i[int pointer], :void,
|
70
|
+
blocking: true
|
71
|
+
else
|
72
|
+
attach_function lib_fname, function[:ffi_types], function[:ffi_ret_type], blocking: true
|
73
|
+
end
|
74
|
+
if block
|
75
|
+
[self, singleton_class].each do |receiver|
|
76
|
+
receiver.prepend(Module.new do
|
77
|
+
define_method(method_name, &block)
|
78
|
+
end)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
[self, singleton_class].each do |receiver|
|
84
|
+
receiver.define_method(method_name) do |*args|
|
85
|
+
CrystalRuby.build! unless CrystalRuby.compiled?
|
86
|
+
unless CrystalRuby.attached?
|
87
|
+
CrystalRuby.attach!
|
88
|
+
return send(method_name, *args) if block
|
89
|
+
end
|
90
|
+
args.each_with_index do |arg, i|
|
91
|
+
args[i] = function[:arg_maps][i][arg] if function[:arg_maps][i]
|
92
|
+
end
|
93
|
+
|
94
|
+
result = Reactor.schedule_work!(self, lib_fname, *args, function[:ffi_ret_type], async: async)
|
95
|
+
|
96
|
+
if function[:retval_map]
|
97
|
+
function[:retval_map][result]
|
98
|
+
else
|
99
|
+
result
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
require "open3"
|
2
|
+
require "tmpdir"
|
3
|
+
require "shellwords"
|
4
|
+
|
5
|
+
module CrystalRuby
|
6
|
+
module Compilation
|
7
|
+
def self.compile!(
|
8
|
+
src: CrystalRuby.config.crystal_src_dir_abs / CrystalRuby.config.crystal_main_file,
|
9
|
+
lib: CrystalRuby.config.crystal_lib_dir_abs / CrystalRuby.config.crystal_lib_name,
|
10
|
+
verbose: CrystalRuby.config.verbose,
|
11
|
+
debug: CrystalRuby.config.debug
|
12
|
+
)
|
13
|
+
Dir.chdir(CrystalRuby.config.crystal_src_dir_abs) do
|
14
|
+
compile_command = compile_command!(verbose: verbose, debug: debug, lib: lib, src: src)
|
15
|
+
link_command = link_cmd!(verbose: verbose, lib: lib, src: src)
|
16
|
+
|
17
|
+
CrystalRuby.log_debug "Compiling Crystal code: #{compile_command}"
|
18
|
+
unless system(compile_command)
|
19
|
+
CrystalRuby.log_error "Failed to build Crystal object file."
|
20
|
+
return false
|
21
|
+
end
|
22
|
+
|
23
|
+
CrystalRuby.log_debug "Linking Crystal code: #{link_command}"
|
24
|
+
unless system(link_command)
|
25
|
+
CrystalRuby.log_error "Failed to link Crystal library."
|
26
|
+
return false
|
27
|
+
end
|
28
|
+
CrystalRuby.log_info "Compilation successful"
|
29
|
+
end
|
30
|
+
|
31
|
+
true
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.compile_command!(verbose:, debug:, lib:, src:)
|
35
|
+
@compile_command ||= begin
|
36
|
+
verbose_flag = verbose ? "--verbose" : ""
|
37
|
+
debug_flag = debug ? "" : "--release --no-debug"
|
38
|
+
redirect_output = " > /dev/null " unless verbose
|
39
|
+
|
40
|
+
src = Shellwords.escape(src)
|
41
|
+
lib = Shellwords.escape(lib)
|
42
|
+
|
43
|
+
%(crystal build #{verbose_flag} #{debug_flag} --cross-compile -o #{lib} #{src}#{redirect_output})
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
# Here we misuse the crystal compiler to build a valid linking command
|
48
|
+
# with all of the platform specific flags that we need.
|
49
|
+
# We then use this command to link the object file that we compiled in the previous step.
|
50
|
+
# This is not robust and is likely to need revision in the future.
|
51
|
+
def self.link_cmd!(verbose:, lib:, src:)
|
52
|
+
@link_cmd ||= begin
|
53
|
+
result = nil
|
54
|
+
|
55
|
+
Dir.mktmpdir do |tmp|
|
56
|
+
CrystalRuby.log_debug "Building link command"
|
57
|
+
src = Shellwords.escape(src)
|
58
|
+
lib_dir = Shellwords.escape(CrystalRuby.config.crystal_src_dir_abs / "lib")
|
59
|
+
escaped_output_path = Shellwords.escape(Pathname.new(tmp) / "main")
|
60
|
+
|
61
|
+
command = "timeout -k 2s 2s bash -c \"export CRYSTAL_PATH=$(crystal env CRYSTAL_PATH):#{lib_dir} && crystal build --verbose #{src} -o #{escaped_output_path} \""
|
62
|
+
|
63
|
+
output = ""
|
64
|
+
pid = nil
|
65
|
+
|
66
|
+
CrystalRuby.log_debug "Running command: #{command}"
|
67
|
+
|
68
|
+
Open3.popen2e(command) do |_stdin, stdout_and_stderr, _wait_thr|
|
69
|
+
while line = stdout_and_stderr.gets
|
70
|
+
puts line if verbose
|
71
|
+
output += line # Capture the output
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
CrystalRuby.log_debug "Parsing link command"
|
76
|
+
|
77
|
+
# Parse the output to find the last invocation of the C compiler, which is likely the linking stage
|
78
|
+
# and strip off the targets that the crystal compiler added.
|
79
|
+
link_command_suffix = output.lines.select { |line| line.strip.start_with?("cc") }.last.strip[/.*(-o.*)/, 1]
|
80
|
+
|
81
|
+
# Replace the output file with the path to the object file we compiled
|
82
|
+
link_command_suffix.gsub!(
|
83
|
+
/-o.*main/,
|
84
|
+
"-o #{lib}"
|
85
|
+
)
|
86
|
+
result = %(cc #{lib}.o -shared #{link_command_suffix})
|
87
|
+
result << " > /dev/null 2>&1" unless verbose
|
88
|
+
result
|
89
|
+
end
|
90
|
+
|
91
|
+
result
|
92
|
+
end
|
93
|
+
rescue StandardError
|
94
|
+
""
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
data/lib/crystalruby/config.rb
CHANGED
@@ -1,34 +1,84 @@
|
|
1
1
|
require "singleton"
|
2
2
|
require "yaml"
|
3
|
+
require "logger"
|
3
4
|
|
4
5
|
module CrystalRuby
|
5
6
|
def self.config
|
6
7
|
Config.instance
|
7
8
|
end
|
8
9
|
|
10
|
+
%w[debug info warn error].each do |level|
|
11
|
+
define_singleton_method("log_#{level}") do |*msg|
|
12
|
+
prefix = config.colorize_log_output ? "\e[33mcrystalruby\e[0m\e[90m [#{Thread.current.object_id}]\e[0m" : "[crystalruby] #{Thread.current.object_id}"
|
13
|
+
|
14
|
+
config.logger.send(level, "#{prefix} #{msg.join(", ")}")
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
9
18
|
# Define a nested Config class
|
10
19
|
class Config
|
11
20
|
include Singleton
|
12
|
-
attr_accessor :debug, :
|
13
|
-
:crystal_lib_name, :crystal_codegen_dir
|
21
|
+
attr_accessor :debug, :verbose, :logger, :colorize_log_output, :single_thread_mode
|
14
22
|
|
15
23
|
def initialize
|
16
24
|
@debug = true
|
25
|
+
@paths_cache = {}
|
17
26
|
config = File.exist?("crystalruby.yaml") && begin
|
18
27
|
YAML.safe_load(IO.read("crystalruby.yaml"))
|
19
28
|
rescue StandardError
|
20
29
|
nil
|
21
30
|
end || {}
|
22
|
-
@crystal_src_dir
|
23
|
-
@crystal_lib_dir
|
24
|
-
@crystal_main_file
|
25
|
-
@crystal_lib_name
|
26
|
-
@crystal_codegen_dir
|
27
|
-
@
|
31
|
+
@crystal_src_dir = config.fetch("crystal_src_dir", "./crystalruby/src")
|
32
|
+
@crystal_lib_dir = config.fetch("crystal_lib_dir", "./crystalruby/lib")
|
33
|
+
@crystal_main_file = config.fetch("crystal_main_file", "main.cr")
|
34
|
+
@crystal_lib_name = config.fetch("crystal_lib_name", "crlib")
|
35
|
+
@crystal_codegen_dir = config.fetch("crystal_codegen_dir", "generated")
|
36
|
+
@crystal_project_root = config.fetch("crystal_project_root", Pathname.pwd)
|
37
|
+
@debug = config.fetch("debug", true)
|
38
|
+
@verbose = config.fetch("verbose", false)
|
39
|
+
@single_thread_mode = config.fetch("single_thread_mode", false)
|
40
|
+
@colorize_log_output = config.fetch("colorize_log_output", false)
|
41
|
+
@log_level = config.fetch("log_level", ENV.fetch("CRYSTALRUBY_LOG_LEVEL", "info"))
|
42
|
+
@logger = Logger.new(STDOUT)
|
43
|
+
@logger.level = Logger.const_get(@log_level.to_s.upcase)
|
44
|
+
end
|
45
|
+
|
46
|
+
%w[crystal_main_file crystal_lib_name crystal_project_root].each do |method_name|
|
47
|
+
define_method(method_name) do
|
48
|
+
@paths_cache[method_name] ||= Pathname.new(instance_variable_get(:"@#{method_name}"))
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
%w[crystal_codegen_dir].each do |method_name|
|
53
|
+
abs_method_name = "#{method_name}_abs"
|
54
|
+
define_method(abs_method_name) do
|
55
|
+
@paths_cache[abs_method_name] ||= crystal_src_dir_abs / instance_variable_get(:"@#{method_name}")
|
56
|
+
end
|
57
|
+
|
58
|
+
define_method(method_name) do
|
59
|
+
@paths_cache[method_name] ||= Pathname.new instance_variable_get(:"@#{method_name}")
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
%w[crystal_src_dir crystal_lib_dir].each do |method_name|
|
64
|
+
abs_method_name = "#{method_name}_abs"
|
65
|
+
define_method(abs_method_name) do
|
66
|
+
@paths_cache[abs_method_name] ||= crystal_project_root / instance_variable_get(:"@#{method_name}")
|
67
|
+
end
|
68
|
+
|
69
|
+
define_method(method_name) do
|
70
|
+
@paths_cache[method_name] ||= Pathname.new instance_variable_get(:"@#{method_name}")
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def log_level=(level)
|
75
|
+
@log_level = level
|
76
|
+
@logger.level = Logger.const_get(level.to_s.upcase)
|
28
77
|
end
|
29
78
|
end
|
30
79
|
|
31
80
|
def self.configure
|
32
81
|
yield(config)
|
82
|
+
@paths_cache = {}
|
33
83
|
end
|
34
84
|
end
|
@@ -0,0 +1,184 @@
|
|
1
|
+
module CrystalRuby
|
2
|
+
module Reactor
|
3
|
+
module_function
|
4
|
+
|
5
|
+
class ReactorStoppedException < StandardError; end
|
6
|
+
class SingleThreadViolation < StandardError; end
|
7
|
+
|
8
|
+
REACTOR_QUEUE = Queue.new
|
9
|
+
|
10
|
+
# We maintain a map of threads, each with a mutex, condition variable, and result
|
11
|
+
THREAD_MAP = Hash.new do |h, tid_or_thread, tid = tid_or_thread|
|
12
|
+
if tid_or_thread.is_a?(Thread)
|
13
|
+
ObjectSpace.define_finalizer(tid_or_thread) do
|
14
|
+
THREAD_MAP.delete(tid_or_thread)
|
15
|
+
THREAD_MAP.delete(tid_or_thread.object_id)
|
16
|
+
end
|
17
|
+
tid = tid_or_thread.object_id
|
18
|
+
end
|
19
|
+
|
20
|
+
h[tid] = {
|
21
|
+
mux: Mutex.new,
|
22
|
+
cond: ConditionVariable.new,
|
23
|
+
result: nil,
|
24
|
+
thread_id: tid
|
25
|
+
}
|
26
|
+
h[tid_or_thread] = h[tid] if tid_or_thread.is_a?(Thread)
|
27
|
+
end
|
28
|
+
|
29
|
+
# We memoize callbacks, once per return type
|
30
|
+
CALLBACKS_MAP = Hash.new do |h, rt|
|
31
|
+
h[rt] = FFI::Function.new(:void, [:int, *(rt == :void ? [] : [rt])]) do |tid, ret|
|
32
|
+
THREAD_MAP[tid][:error] = nil
|
33
|
+
THREAD_MAP[tid][:result] = ret
|
34
|
+
THREAD_MAP[tid][:cond].signal
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
ERROR_CALLBACK = FFI::Function.new(:void, %i[string string int]) do |error_type, message, tid|
|
39
|
+
error_type = error_type.to_sym
|
40
|
+
is_exception_type = Object.const_defined?(error_type) && Object.const_get(error_type).ancestors.include?(Exception)
|
41
|
+
error_type = is_exception_type ? Object.const_get(error_type) : RuntimeError
|
42
|
+
tid = tid.zero? ? Reactor.current_thread_id : tid
|
43
|
+
THREAD_MAP[tid][:error] = error_type.new(message)
|
44
|
+
THREAD_MAP[tid][:result] = nil
|
45
|
+
THREAD_MAP[tid][:cond].signal
|
46
|
+
end
|
47
|
+
|
48
|
+
def thread_conditions
|
49
|
+
THREAD_MAP[Thread.current]
|
50
|
+
end
|
51
|
+
|
52
|
+
def await_result!
|
53
|
+
mux, cond = thread_conditions.values_at(:mux, :cond)
|
54
|
+
cond.wait(mux)
|
55
|
+
raise THREAD_MAP[thread_id][:error] if THREAD_MAP[thread_id][:error]
|
56
|
+
|
57
|
+
THREAD_MAP[thread_id][:result]
|
58
|
+
end
|
59
|
+
|
60
|
+
def thread_id
|
61
|
+
Thread.current.object_id
|
62
|
+
end
|
63
|
+
|
64
|
+
def yield!(time: 0)
|
65
|
+
Thread.new do
|
66
|
+
sleep time
|
67
|
+
schedule_work!(Reactor, :yield, nil, async: false, blocking: false)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def current_thread_id=(val)
|
72
|
+
@current_thread_id = val
|
73
|
+
end
|
74
|
+
|
75
|
+
def current_thread_id
|
76
|
+
@current_thread_id
|
77
|
+
end
|
78
|
+
|
79
|
+
def schedule_work!(receiver, op_name, *args, return_type, blocking: true, async: true)
|
80
|
+
raise ReactorStoppedException, "Reactor has been terminated, no new work can be scheduled" if @stopped
|
81
|
+
|
82
|
+
if @single_thread_mode
|
83
|
+
unless Thread.current.object_id == @main_thread_id
|
84
|
+
raise SingleThreadViolation,
|
85
|
+
"Single thread mode is enabled, cannot run in multi-threaded mode. " \
|
86
|
+
"Reactor was started from: #{@main_thread_id}, then called from #{Thread.current.object_id}"
|
87
|
+
end
|
88
|
+
|
89
|
+
return receiver.send(op_name, *args)
|
90
|
+
end
|
91
|
+
|
92
|
+
tvars = thread_conditions
|
93
|
+
tvars[:mux].synchronize do
|
94
|
+
REACTOR_QUEUE.push(
|
95
|
+
case true
|
96
|
+
when async
|
97
|
+
lambda {
|
98
|
+
receiver.send(
|
99
|
+
op_name, *args, tvars[:thread_id],
|
100
|
+
CALLBACKS_MAP[return_type]
|
101
|
+
)
|
102
|
+
yield!(time: 0)
|
103
|
+
}
|
104
|
+
when blocking
|
105
|
+
lambda {
|
106
|
+
tvars[:error] = nil
|
107
|
+
Reactor.current_thread_id = tvars[:thread_id]
|
108
|
+
begin
|
109
|
+
result = receiver.send(op_name, *args)
|
110
|
+
rescue StandardError => e
|
111
|
+
tvars[:error] = e
|
112
|
+
end
|
113
|
+
tvars[:result] = result unless tvars[:error]
|
114
|
+
tvars[:cond].signal
|
115
|
+
}
|
116
|
+
else
|
117
|
+
lambda {
|
118
|
+
outstanding_jobs = receiver.send(op_name, *args)
|
119
|
+
yield!(time: 0.01) unless outstanding_jobs.zero?
|
120
|
+
}
|
121
|
+
end
|
122
|
+
)
|
123
|
+
return await_result! if blocking
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def init_single_thread_mode!
|
128
|
+
@single_thread_mode = true
|
129
|
+
@main_thread_id = Thread.current.object_id
|
130
|
+
init_crystal_ruby!
|
131
|
+
end
|
132
|
+
|
133
|
+
def init_crystal_ruby!
|
134
|
+
attach_lib!
|
135
|
+
init(ERROR_CALLBACK)
|
136
|
+
end
|
137
|
+
|
138
|
+
def attach_lib!
|
139
|
+
CrystalRuby.log_debug("Attaching lib")
|
140
|
+
extend FFI::Library
|
141
|
+
ffi_lib CrystalRuby.config.crystal_lib_dir / CrystalRuby.config.crystal_lib_name
|
142
|
+
attach_function :init, [:pointer], :void
|
143
|
+
attach_function :stop, [], :void
|
144
|
+
attach_function :yield, %i[], :int
|
145
|
+
end
|
146
|
+
|
147
|
+
def stop!
|
148
|
+
CrystalRuby.log_debug("Stopping reactor")
|
149
|
+
@stopped = true
|
150
|
+
sleep 1
|
151
|
+
@main_loop&.kill
|
152
|
+
@main_loop = nil
|
153
|
+
CrystalRuby.log_debug("Reactor stopped")
|
154
|
+
end
|
155
|
+
|
156
|
+
def running?
|
157
|
+
@main_loop&.alive?
|
158
|
+
end
|
159
|
+
|
160
|
+
def start!
|
161
|
+
@main_loop ||= begin
|
162
|
+
attach_lib!
|
163
|
+
Thread.new do
|
164
|
+
CrystalRuby.log_debug("Starting reactor")
|
165
|
+
init(ERROR_CALLBACK)
|
166
|
+
CrystalRuby.log_debug("CrystalRuby initialized")
|
167
|
+
loop do
|
168
|
+
REACTOR_QUEUE.pop[]
|
169
|
+
break if @stopped
|
170
|
+
end
|
171
|
+
stop
|
172
|
+
CrystalRuby.log_debug("Stopping reactor")
|
173
|
+
rescue StandardError => e
|
174
|
+
puts "Error: #{e}"
|
175
|
+
puts e.backtrace
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
at_exit do
|
181
|
+
@stopped = true
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
data/lib/crystalruby/template.rb
CHANGED
@@ -2,11 +2,12 @@ module CrystalRuby
|
|
2
2
|
module Template
|
3
3
|
Dir[File.join(File.dirname(__FILE__), "templates", "*.cr")].each do |file|
|
4
4
|
template_name = File.basename(file, File.extname(file)).split("_").map(&:capitalize).join
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
5
|
+
template_value = File.read(file)
|
6
|
+
template_value.define_singleton_method(:render) do |context|
|
7
|
+
CrystalRuby.log_debug("Template.render: #{template_name}")
|
8
|
+
self % context
|
9
|
+
end
|
10
|
+
const_set(template_name, template_value)
|
10
11
|
end
|
11
12
|
end
|
12
13
|
end
|
@@ -12,7 +12,6 @@ end
|
|
12
12
|
# This function is the entry point for the CrystalRuby code, exposed through FFI.
|
13
13
|
# We apply some basic error handling here, and convert the arguments and return values
|
14
14
|
# to ensure that we are using Crystal native types.
|
15
|
-
|
16
15
|
fun %{lib_fn_name}(%{lib_fn_args}): %{lib_fn_ret_type}
|
17
16
|
begin
|
18
17
|
%{convert_lib_args}
|
@@ -20,10 +19,44 @@ fun %{lib_fn_name}(%{lib_fn_args}): %{lib_fn_ret_type}
|
|
20
19
|
return_value = %{module_name}.%{fn_name}(%{arg_names})
|
21
20
|
return %{convert_return_type}
|
22
21
|
rescue ex
|
23
|
-
CrystalRuby.report_error("RuntimeError", ex.message.to_s)
|
22
|
+
CrystalRuby.report_error("RuntimeError", ex.message.to_s, 0)
|
24
23
|
end
|
25
24
|
rescue ex
|
26
|
-
CrystalRuby.report_error("ArgumentError", ex.message.to_s)
|
25
|
+
CrystalRuby.report_error("ArgumentError", ex.message.to_s, 0)
|
27
26
|
end
|
28
27
|
return %{error_value}
|
29
28
|
end
|
29
|
+
|
30
|
+
|
31
|
+
# This function is the async entry point for the CrystalRuby code, exposed through FFI.
|
32
|
+
# We apply some basic error handling here, and convert the arguments and return values
|
33
|
+
# to ensure that we are using Crystal native types.
|
34
|
+
fun %{lib_fn_name}_async(%{lib_fn_args} thread_id: UInt32, callback : %{callback_type}): Void
|
35
|
+
begin
|
36
|
+
%{convert_lib_args}
|
37
|
+
CrystalRuby.increment_task_counter
|
38
|
+
spawn do
|
39
|
+
begin
|
40
|
+
return_value = %{module_name}.%{fn_name}(%{arg_names})
|
41
|
+
converted = %{convert_return_type}
|
42
|
+
CrystalRuby.queue_callback(->{
|
43
|
+
%{callback_call}
|
44
|
+
CrystalRuby.decrement_task_counter
|
45
|
+
})
|
46
|
+
rescue ex
|
47
|
+
exception = ex.message.to_s
|
48
|
+
CrystalRuby.queue_callback(->{
|
49
|
+
CrystalRuby.error_callback.call("RuntimeError".to_unsafe, exception.to_unsafe, thread_id)
|
50
|
+
CrystalRuby.decrement_task_counter
|
51
|
+
})
|
52
|
+
end
|
53
|
+
end
|
54
|
+
rescue ex
|
55
|
+
|
56
|
+
exception = ex.message.to_s
|
57
|
+
CrystalRuby.queue_callback(->{
|
58
|
+
CrystalRuby.error_callback.call("RuntimeError".to_unsafe, ex.message.to_s.to_unsafe, thread_id)
|
59
|
+
CrystalRuby.decrement_task_counter
|
60
|
+
})
|
61
|
+
end
|
62
|
+
end
|
@@ -1,16 +1,24 @@
|
|
1
|
-
|
1
|
+
alias ErrorCallback = (Pointer(UInt8), Pointer(UInt8), UInt32 -> Void)
|
2
2
|
|
3
|
-
|
3
|
+
ARGV1 = "crystalruby"
|
4
|
+
CALLBACK_MUX = Mutex.new
|
4
5
|
|
5
6
|
module CrystalRuby
|
6
7
|
# Initializing Crystal Ruby invokes init on the Crystal garbage collector.
|
7
8
|
# We need to be sure to only do this once.
|
8
9
|
@@initialized = false
|
9
10
|
|
10
|
-
#
|
11
|
-
#
|
12
|
-
|
13
|
-
|
11
|
+
# Our Ruby <-> Crystal Reactor uses Fibers, with callbacks to allow
|
12
|
+
# multiple concurrent Crystal operations to be queued
|
13
|
+
@@callbacks = [] of Proc(Nil)
|
14
|
+
|
15
|
+
# We only continue to yield to the Crystal scheduler from Ruby
|
16
|
+
# while there are outstanding tasks.
|
17
|
+
@@task_counter : Atomic(Int32) = Atomic.new(0)
|
18
|
+
|
19
|
+
# We can override the error callback to catch errors in Crystal,
|
20
|
+
# and explicitly expose them to Ruby.
|
21
|
+
@@error_callback : ErrorCallback = ->(t : UInt8* , s : UInt8*, tid : UInt32){ puts "Error: #{t}:#{s}" }
|
14
22
|
|
15
23
|
# This is the entry point for instantiating CrystalRuby
|
16
24
|
# We:
|
@@ -19,28 +27,95 @@ module CrystalRuby
|
|
19
27
|
# 3. Call the Crystal main function
|
20
28
|
def self.init(error_callback : ErrorCallback)
|
21
29
|
return if @@initialized
|
30
|
+
@@initialized = true
|
22
31
|
GC.init
|
23
|
-
|
32
|
+
argv_ptr = ARGV1.to_unsafe
|
33
|
+
Crystal.main(0, pointerof(argv_ptr))
|
24
34
|
@@error_callback = error_callback
|
25
|
-
ptr = FAKE_ARG.to_unsafe
|
26
|
-
LibCrystalMain.__crystal_main(1, pointerof(ptr))
|
27
35
|
end
|
28
36
|
|
29
37
|
# Explicit error handling (triggers exception within Ruby on the same thread)
|
30
|
-
def self.report_error(error_type : String, str : String)
|
31
|
-
|
32
|
-
|
38
|
+
def self.report_error(error_type : String, str : String, thread_id : UInt32, )
|
39
|
+
@@error_callback.call(error_type.to_unsafe, str.to_unsafe, thread_id)
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.error_callback : ErrorCallback
|
43
|
+
@@error_callback
|
44
|
+
end
|
45
|
+
|
46
|
+
# New async task started
|
47
|
+
def self.increment_task_counter
|
48
|
+
@@task_counter.add(1)
|
49
|
+
end
|
50
|
+
|
51
|
+
# Async task finished
|
52
|
+
def self.decrement_task_counter
|
53
|
+
@@task_counter.sub(1)
|
54
|
+
end
|
55
|
+
|
56
|
+
# Get number of outstanding tasks
|
57
|
+
def self.get_task_counter : Int32
|
58
|
+
@@task_counter.get()
|
59
|
+
end
|
60
|
+
|
61
|
+
# Queue a callback for an async task
|
62
|
+
def self.queue_callback(callback : Proc(Nil))
|
63
|
+
CALLBACK_MUX.synchronize do
|
64
|
+
@@callbacks << callback
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
# Get number of queued callbacks
|
69
|
+
def self.count_callbacks : Int32
|
70
|
+
@@callbacks.size
|
71
|
+
end
|
72
|
+
|
73
|
+
# Flush all callbacks
|
74
|
+
def self.flush_callbacks : Int32
|
75
|
+
CALLBACK_MUX.synchronize do
|
76
|
+
count = @@callbacks.size
|
77
|
+
@@callbacks.each do |callback|
|
78
|
+
result = callback.call()
|
79
|
+
end
|
80
|
+
@@callbacks.clear
|
33
81
|
end
|
82
|
+
get_task_counter
|
34
83
|
end
|
35
84
|
end
|
36
85
|
|
86
|
+
# Initialize CrystalRuby
|
37
87
|
fun init(cb : ErrorCallback): Void
|
38
88
|
CrystalRuby.init(cb)
|
39
89
|
end
|
40
90
|
|
91
|
+
fun stop(): Void
|
92
|
+
GC.disable
|
93
|
+
end
|
94
|
+
|
95
|
+
# Yield to the Crystal scheduler from Ruby
|
96
|
+
# If there's callbacks to process, we flush them
|
97
|
+
# Otherwise, we yield to the Crystal scheduler and let Ruby know
|
98
|
+
# how many outstanding tasks still remain (it will stop yielding to Crystal
|
99
|
+
# once this figure reaches 0).
|
100
|
+
fun yield() : Int32
|
101
|
+
if CrystalRuby.count_callbacks == 0
|
102
|
+
|
103
|
+
Fiber.yield
|
104
|
+
|
105
|
+
# TODO: We should apply backpressure here to prevent busy waiting if the number of outstanding tasks is not decreasing.
|
106
|
+
# Use a simple exponential backoff strategy, to increase the time between each yield up to a maximum of 1 second.
|
107
|
+
|
108
|
+
CrystalRuby.get_task_counter
|
109
|
+
else
|
110
|
+
CrystalRuby.flush_callbacks()
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
|
41
115
|
# This is where we define all our Crystal modules and types
|
42
116
|
# derived from their Ruby counterparts.
|
43
117
|
%{type_modules}
|
44
118
|
|
45
119
|
# Require all generated crystal files
|
120
|
+
require "json"
|
46
121
|
%{requires}
|
data/lib/crystalruby/typemaps.rb
CHANGED
@@ -52,14 +52,19 @@ module CrystalRuby
|
|
52
52
|
string: '"".to_unsafe' # String type
|
53
53
|
}
|
54
54
|
|
55
|
-
C_TYPE_MAP = CRYSTAL_TYPE_MAP.merge(
|
56
|
-
|
57
|
-
|
55
|
+
C_TYPE_MAP = CRYSTAL_TYPE_MAP.merge(
|
56
|
+
{
|
57
|
+
string: "Pointer(UInt8)"
|
58
|
+
}
|
59
|
+
)
|
58
60
|
|
59
61
|
C_TYPE_CONVERSIONS = {
|
60
62
|
string: {
|
61
63
|
from: "String.new(%s)",
|
62
64
|
to: "%s.to_unsafe"
|
65
|
+
},
|
66
|
+
void: {
|
67
|
+
to: "nil"
|
63
68
|
}
|
64
69
|
}
|
65
70
|
end
|
data/lib/crystalruby/version.rb
CHANGED
data/lib/crystalruby.rb
CHANGED
@@ -4,121 +4,46 @@ require "ffi"
|
|
4
4
|
require "digest"
|
5
5
|
require "fileutils"
|
6
6
|
require "method_source"
|
7
|
+
require "pathname"
|
8
|
+
|
7
9
|
require_relative "crystalruby/config"
|
8
10
|
require_relative "crystalruby/version"
|
9
11
|
require_relative "crystalruby/typemaps"
|
10
12
|
require_relative "crystalruby/types"
|
11
13
|
require_relative "crystalruby/typebuilder"
|
12
14
|
require_relative "crystalruby/template"
|
15
|
+
require_relative "crystalruby/compilation"
|
16
|
+
require_relative "crystalruby/adapter"
|
17
|
+
require_relative "crystalruby/reactor"
|
13
18
|
|
14
19
|
module CrystalRuby
|
15
20
|
CR_SRC_FILES_PATTERN = "./**/*.cr"
|
16
|
-
|
17
|
-
def crystalize(type = :src, **options, &block)
|
18
|
-
(args,), returns = options.first
|
19
|
-
args ||= {}
|
20
|
-
raise "Arguments should be of the form name: :type. Got #{args}" unless args.is_a?(Hash)
|
21
|
-
|
22
|
-
@crystalize_next = { raw: type.to_sym == :raw, args: args, returns: returns, block: block }
|
23
|
-
end
|
24
|
-
|
25
|
-
def crystal(type = :src, &block)
|
26
|
-
inline_crystal_body = Template.render(
|
27
|
-
Template::InlineChunk,
|
28
|
-
{
|
29
|
-
module_name: name,
|
30
|
-
body: block.source.lines[
|
31
|
-
type == :raw ? 2...-2 : 1...-1
|
32
|
-
].join("\n")
|
33
|
-
}
|
34
|
-
)
|
35
|
-
CrystalRuby.write_chunk(self, body: inline_crystal_body)
|
36
|
-
end
|
37
|
-
|
38
|
-
def crtype(&block)
|
39
|
-
TypeBuilder.with_injected_type_dsl(self) do
|
40
|
-
TypeBuilder.build(&block)
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
def json(&block)
|
45
|
-
crtype(&block).serialize_as(:json)
|
46
|
-
end
|
47
|
-
|
48
|
-
def method_added(method_name)
|
49
|
-
if @crystalize_next
|
50
|
-
attach_crystalized_method(method_name)
|
51
|
-
@crystalize_next = nil
|
52
|
-
end
|
53
|
-
super
|
54
|
-
end
|
55
|
-
|
56
|
-
def config
|
57
|
-
CrystalRuby.config
|
58
|
-
end
|
59
|
-
|
60
|
-
def attach_crystalized_method(method_name)
|
61
|
-
CrystalRuby.instantiate_crystal_ruby! unless CrystalRuby.instantiated?
|
62
|
-
|
63
|
-
function_body = instance_method(method_name).source.lines[
|
64
|
-
@crystalize_next[:raw] ? 2...-2 : 1...-1
|
65
|
-
].join("\n")
|
66
|
-
|
67
|
-
fname = "#{name.downcase}_#{method_name}"
|
68
|
-
args, returns, block = @crystalize_next.values_at(:args, :returns, :block)
|
69
|
-
args ||= {}
|
70
|
-
@crystalize_next = nil
|
71
|
-
function = build_function(self, method_name, args, returns, function_body)
|
72
|
-
CrystalRuby.write_chunk(self, name: function[:name], body: function[:body]) do
|
73
|
-
extend FFI::Library
|
74
|
-
ffi_lib "#{config.crystal_lib_dir}/#{config.crystal_lib_name}"
|
75
|
-
attach_function method_name, fname, function[:ffi_types], function[:return_ffi_type]
|
76
|
-
if block
|
77
|
-
[singleton_class, self].each do |receiver|
|
78
|
-
receiver.prepend(Module.new do
|
79
|
-
define_method(method_name, &block)
|
80
|
-
end)
|
81
|
-
end
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
|
-
[singleton_class, self].each do |receiver|
|
86
|
-
receiver.prepend(Module.new do
|
87
|
-
define_method(method_name) do |*args|
|
88
|
-
CrystalRuby.compile! unless CrystalRuby.compiled?
|
89
|
-
unless CrystalRuby.attached?
|
90
|
-
CrystalRuby.attach!
|
91
|
-
return send(method_name, *args) if block
|
92
|
-
end
|
93
|
-
args.each_with_index do |arg, i|
|
94
|
-
args[i] = function[:arg_maps][i][arg] if function[:arg_maps][i]
|
95
|
-
end
|
96
|
-
result = super(*args)
|
97
|
-
if function[:retval_map]
|
98
|
-
function[:retval_map][result]
|
99
|
-
else
|
100
|
-
result
|
101
|
-
end
|
102
|
-
end
|
103
|
-
end)
|
104
|
-
end
|
105
|
-
end
|
21
|
+
CR_COMPILE_MUX = Mutex.new
|
106
22
|
|
107
23
|
module_function
|
108
24
|
|
109
|
-
def build_function(owner, name, args, returns, body)
|
25
|
+
def build_function(owner, lib_fn_name, name, args, returns, body)
|
26
|
+
log_debug(".build_function #{{ owner: owner, name: name, args: args, returns: returns, body: body[0..50] }}")
|
27
|
+
|
110
28
|
arg_types = args.transform_values(&method(:build_type_map))
|
111
29
|
return_type = build_type_map(returns)
|
112
|
-
|
113
|
-
|
30
|
+
lib_fn_args = arg_types.map { |k, arg_type| "_#{k} : #{arg_type[:lib_type]}" }.join(",")
|
31
|
+
lib_fn_args += ", " unless lib_fn_args.empty?
|
32
|
+
lib_fn_arg_names = arg_types.map { |k, _arg_type| "_#{k}" }.join(",")
|
33
|
+
lib_fn_arg_names += ", " unless lib_fn_args.empty?
|
34
|
+
|
35
|
+
function_body = Template::Function.render(
|
114
36
|
{
|
115
37
|
module_name: owner.name,
|
116
|
-
lib_fn_name:
|
38
|
+
lib_fn_name: lib_fn_name,
|
117
39
|
fn_name: name,
|
118
40
|
fn_body: body,
|
41
|
+
callback_call: returns == :void ? "callback.call(thread_id)" : "callback.call(thread_id, converted)",
|
42
|
+
callback_type: return_type[:ffi_type] == :void ? "UInt32 -> Void" : " UInt32, #{return_type[:lib_type]} -> Void",
|
119
43
|
fn_args: arg_types.map { |k, arg_type| "#{k} : #{arg_type[:crystal_type]}" }.join(","),
|
120
44
|
fn_ret_type: return_type[:crystal_type],
|
121
|
-
lib_fn_args:
|
45
|
+
lib_fn_args: lib_fn_args,
|
46
|
+
lib_fn_arg_names: lib_fn_arg_names,
|
122
47
|
lib_fn_ret_type: return_type[:lib_type],
|
123
48
|
convert_lib_args: arg_types.map do |k, arg_type|
|
124
49
|
"#{k} = #{arg_type[:convert_lib_to_crystal_type]["_#{k}"]}"
|
@@ -134,7 +59,7 @@ module CrystalRuby
|
|
134
59
|
retval_map: returns.is_a?(Types::TypeSerializer) ? ->(rv) { returns.prepare_retval(rv) } : nil,
|
135
60
|
ffi_types: arg_types.map { |_k, arg_type| arg_type[:ffi_type] },
|
136
61
|
arg_maps: arg_types.map { |_k, arg_type| arg_type[:mapper] },
|
137
|
-
|
62
|
+
ffi_ret_type: return_type[:ffi_ret_type]
|
138
63
|
}
|
139
64
|
end
|
140
65
|
|
@@ -145,7 +70,7 @@ module CrystalRuby
|
|
145
70
|
|
146
71
|
{
|
147
72
|
ffi_type: ffi_type(crystalruby_type),
|
148
|
-
|
73
|
+
ffi_ret_type: ffi_type(crystalruby_type),
|
149
74
|
crystal_type: crystal_type(crystalruby_type),
|
150
75
|
lib_type: lib_type(crystalruby_type),
|
151
76
|
error_value: error_value(crystalruby_type),
|
@@ -219,37 +144,22 @@ module CrystalRuby
|
|
219
144
|
raise "Missing config option `#{config_key}`. \nProvide this inside crystalruby.yaml (run `bundle exec crystalruby init` to generate this file with detaults)"
|
220
145
|
end
|
221
146
|
end
|
222
|
-
FileUtils.mkdir_p
|
223
|
-
FileUtils.mkdir_p
|
224
|
-
|
147
|
+
FileUtils.mkdir_p config.crystal_codegen_dir_abs
|
148
|
+
FileUtils.mkdir_p config.crystal_lib_dir_abs
|
149
|
+
FileUtils.mkdir_p config.crystal_src_dir_abs
|
150
|
+
unless File.exist?(config.crystal_src_dir_abs / config.crystal_main_file)
|
225
151
|
IO.write(
|
226
|
-
|
152
|
+
config.crystal_src_dir_abs / config.crystal_main_file,
|
227
153
|
"require \"./#{config.crystal_codegen_dir}/index\"\n"
|
228
154
|
)
|
229
155
|
end
|
230
156
|
|
231
|
-
|
157
|
+
return if File.exist?(config.crystal_src_dir / "shard.yml")
|
232
158
|
|
233
|
-
|
234
|
-
|
235
|
-
IO.write("#{config.crystal_src_dir}/shard.yml", <<~CRYSTAL)
|
159
|
+
IO.write("#{config.crystal_src_dir}/shard.yml", <<~YAML)
|
236
160
|
name: src
|
237
161
|
version: 0.1.0
|
238
|
-
|
239
|
-
end
|
240
|
-
|
241
|
-
def attach_crystal_ruby_lib!
|
242
|
-
extend FFI::Library
|
243
|
-
ffi_lib "#{config.crystal_lib_dir}/#{config.crystal_lib_name}"
|
244
|
-
attach_function "init!", :init, [:pointer], :void
|
245
|
-
send(:remove_const, :ErrorCallback) if defined?(ErrorCallback)
|
246
|
-
const_set(:ErrorCallback, FFI::Function.new(:void, %i[string string]) do |error_type, message|
|
247
|
-
error_type = error_type.to_sym
|
248
|
-
is_exception_type = Object.const_defined?(error_type) && Object.const_get(error_type).ancestors.include?(Exception)
|
249
|
-
error_type = is_exception_type ? Object.const_get(error_type) : RuntimeError
|
250
|
-
raise error_type.new(message)
|
251
|
-
end)
|
252
|
-
init!(ErrorCallback)
|
162
|
+
YAML
|
253
163
|
end
|
254
164
|
|
255
165
|
def self.instantiated?
|
@@ -272,24 +182,19 @@ module CrystalRuby
|
|
272
182
|
|
273
183
|
def type_modules
|
274
184
|
(@types_cache || {}).map do |type_name, expr|
|
275
|
-
typedef = ""
|
276
185
|
parts = type_name.split("::")
|
277
|
-
|
278
|
-
|
279
|
-
typedef << "#{indent} module #{part}\n"
|
280
|
-
indent += " "
|
186
|
+
typedef = parts[0...-1].each_with_index.reduce("") do |acc, (part, index)|
|
187
|
+
acc + "#{" " * index}module #{part}\n"
|
281
188
|
end
|
282
|
-
typedef
|
283
|
-
parts[0...-1].
|
284
|
-
|
285
|
-
typedef << "#{indent} end\n"
|
189
|
+
typedef += "#{" " * (parts.size - 1)}alias #{parts.last} = #{expr}\n"
|
190
|
+
typedef + parts[0...-1].reverse.each_with_index.reduce("") do |acc, (_part, index)|
|
191
|
+
acc + "#{" " * (parts.size - 2 - index)}end\n"
|
286
192
|
end
|
287
|
-
typedef
|
288
193
|
end.join("\n")
|
289
194
|
end
|
290
195
|
|
291
196
|
def self.requires
|
292
|
-
|
197
|
+
chunk_store.map do |function|
|
293
198
|
function_data = function[:body]
|
294
199
|
file_digest = Digest::MD5.hexdigest function_data
|
295
200
|
fname = function[:name]
|
@@ -297,42 +202,39 @@ module CrystalRuby
|
|
297
202
|
end.join("\n")
|
298
203
|
end
|
299
204
|
|
300
|
-
def self.
|
301
|
-
|
205
|
+
def self.build!
|
206
|
+
log_debug(".build!")
|
302
207
|
|
303
|
-
|
304
|
-
|
305
|
-
|
208
|
+
CR_COMPILE_MUX.synchronize do
|
209
|
+
return if @compiled
|
210
|
+
|
211
|
+
File.write config.crystal_codegen_dir_abs / "index.cr", Template::Index.render(
|
306
212
|
type_modules: type_modules,
|
307
213
|
requires: requires
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
cmd = if config.debug
|
316
|
-
"crystal build -o #{lib_target} #{config.crystal_main_file}"
|
317
|
-
else
|
318
|
-
"crystal build --release --no-debug -o #{lib_target} #{config.crystal_main_file}"
|
319
|
-
end
|
320
|
-
|
321
|
-
unless result = system(cmd)
|
214
|
+
)
|
215
|
+
if @compiled = CrystalRuby::Compilation.compile!(
|
216
|
+
verbose: config.verbose,
|
217
|
+
debug: config.debug
|
218
|
+
)
|
219
|
+
IO.write(digest_file_name, get_cr_src_files_digest)
|
220
|
+
else
|
322
221
|
File.delete(digest_file_name) if File.exist?(digest_file_name)
|
323
222
|
raise "Error compiling crystal code"
|
324
223
|
end
|
325
224
|
end
|
326
|
-
|
327
|
-
IO.write(digest_file_name, get_cr_src_files_digest)
|
328
|
-
@compiled = true
|
329
|
-
attach_crystal_ruby_lib!
|
330
225
|
end
|
331
226
|
|
332
227
|
def self.attach!
|
333
|
-
|
228
|
+
log_debug(".attach!")
|
229
|
+
@chunk_store.each do |function|
|
334
230
|
function[:compile_callback]&.call
|
335
231
|
end
|
232
|
+
log_debug(".attach_crystal_ruby_lib. Single thread mode: #{config.single_thread_mode}")
|
233
|
+
if config.single_thread_mode
|
234
|
+
Reactor.init_single_thread_mode!
|
235
|
+
else
|
236
|
+
Reactor.start!
|
237
|
+
end
|
336
238
|
@attached = true
|
337
239
|
end
|
338
240
|
|
@@ -345,7 +247,11 @@ module CrystalRuby
|
|
345
247
|
end
|
346
248
|
|
347
249
|
def self.digest_file_name
|
348
|
-
@digest_file_name ||= "#{config.
|
250
|
+
@digest_file_name ||= config.crystal_lib_dir_abs / "#{config.crystal_lib_name}.digest"
|
251
|
+
end
|
252
|
+
|
253
|
+
def self.chunk_store
|
254
|
+
@chunk_store ||= []
|
349
255
|
end
|
350
256
|
|
351
257
|
def self.get_current_crystal_lib_digest
|
@@ -353,26 +259,30 @@ module CrystalRuby
|
|
353
259
|
end
|
354
260
|
|
355
261
|
def self.write_chunk(owner, body:, name: Digest::MD5.hexdigest(body), &compile_callback)
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
262
|
+
log_debug(".write_chunk!")
|
263
|
+
chunk_store.delete_if { |chnk| chnk[:owner].name == owner.name && chnk[:name] == name }
|
264
|
+
chunk_store << { owner: owner, name: name, body: body, compile_callback: compile_callback }
|
265
|
+
FileUtils.mkdir_p(config.crystal_codegen_dir_abs)
|
266
|
+
existing = Dir.glob("#{config.crystal_codegen_dir_abs}/**/*.cr")
|
267
|
+
chunk_store.each do |function|
|
268
|
+
log_debug(".processing_chunk", function[0..60])
|
361
269
|
owner_name = function[:owner].name
|
362
|
-
FileUtils.mkdir_p(
|
270
|
+
FileUtils.mkdir_p(config.crystal_codegen_dir_abs / owner_name)
|
363
271
|
function_data = function[:body]
|
364
272
|
fname = function[:name]
|
365
273
|
file_digest = Digest::MD5.hexdigest function_data
|
366
|
-
filename =
|
367
|
-
|
368
|
-
|
274
|
+
filename = config.crystal_codegen_dir_abs / owner_name / "#{fname}_#{file_digest}.cr"
|
275
|
+
|
276
|
+
unless existing.delete(filename.to_s)
|
277
|
+
log_debug("Chunk invalidated", filename.to_s)
|
369
278
|
@attached = false
|
279
|
+
@compiled = false
|
370
280
|
File.write(filename, function_data)
|
371
281
|
end
|
372
282
|
existing.select do |f|
|
373
|
-
f =~
|
283
|
+
f =~ /#{config.crystal_codegen_dir / owner_name / "#{fname}_[a-f0-9]{32}\.cr"}/
|
374
284
|
end.each do |fl|
|
375
|
-
File.delete(fl) unless fl.eql?(filename)
|
285
|
+
File.delete(fl) unless fl.eql?(filename.to_s)
|
376
286
|
end
|
377
287
|
end
|
378
288
|
end
|
data/lib/module.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: crystalruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.13
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Wouter Coppieters
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-04-
|
11
|
+
date: 2024-04-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: digest
|
@@ -77,13 +77,18 @@ files:
|
|
77
77
|
- ".rubocop.yml"
|
78
78
|
- CHANGELOG.md
|
79
79
|
- CODE_OF_CONDUCT.md
|
80
|
+
- Dockerfile
|
80
81
|
- LICENSE.txt
|
81
82
|
- README.md
|
82
83
|
- Rakefile
|
83
84
|
- crystalruby.gemspec
|
85
|
+
- examples/adder/adder.rb
|
84
86
|
- exe/crystalruby
|
85
87
|
- lib/crystalruby.rb
|
88
|
+
- lib/crystalruby/adapter.rb
|
89
|
+
- lib/crystalruby/compilation.rb
|
86
90
|
- lib/crystalruby/config.rb
|
91
|
+
- lib/crystalruby/reactor.rb
|
87
92
|
- lib/crystalruby/template.rb
|
88
93
|
- lib/crystalruby/templates/function.cr
|
89
94
|
- lib/crystalruby/templates/index.cr
|