ruby_wasm 2.5.0 → 2.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/CONTRIBUTING.md +7 -7
  3. data/Cargo.lock +91 -6
  4. data/Gemfile +1 -1
  5. data/README.md +9 -9
  6. data/Rakefile +8 -7
  7. data/docs/cheat_sheet.md +8 -8
  8. data/ext/ruby_wasm/Cargo.toml +3 -1
  9. data/ext/ruby_wasm/src/lib.rs +198 -8
  10. data/lib/ruby_wasm/build/executor.rb +4 -0
  11. data/lib/ruby_wasm/build/product/crossruby.rb +53 -23
  12. data/lib/ruby_wasm/build/product/libyaml.rb +5 -3
  13. data/lib/ruby_wasm/build/product/openssl.rb +7 -2
  14. data/lib/ruby_wasm/build/product/product.rb +3 -3
  15. data/lib/ruby_wasm/build/product/ruby_source.rb +3 -3
  16. data/lib/ruby_wasm/build/product/wasi_vfs.rb +1 -1
  17. data/lib/ruby_wasm/build/product/zlib.rb +3 -1
  18. data/lib/ruby_wasm/build/target.rb +24 -0
  19. data/lib/ruby_wasm/build/toolchain.rb +1 -1
  20. data/lib/ruby_wasm/build.rb +7 -3
  21. data/lib/ruby_wasm/cli.rb +147 -11
  22. data/lib/ruby_wasm/feature_set.rb +30 -0
  23. data/lib/ruby_wasm/packager/component_adapter/wasi_snapshot_preview1.command.wasm +0 -0
  24. data/lib/ruby_wasm/packager/component_adapter/wasi_snapshot_preview1.reactor.wasm +0 -0
  25. data/lib/ruby_wasm/packager/component_adapter.rb +14 -0
  26. data/lib/ruby_wasm/packager/core.rb +192 -4
  27. data/lib/ruby_wasm/packager/file_system.rb +5 -3
  28. data/lib/ruby_wasm/packager.rb +21 -83
  29. data/lib/ruby_wasm/rake_task.rb +1 -0
  30. data/lib/ruby_wasm/version.rb +1 -1
  31. data/lib/ruby_wasm.rb +2 -0
  32. data/package-lock.json +410 -133
  33. data/package.json +3 -3
  34. data/rakelib/ci.rake +3 -3
  35. data/rakelib/packaging.rake +26 -12
  36. data/sig/ruby_wasm/build.rbs +36 -27
  37. data/sig/ruby_wasm/cli.rbs +27 -3
  38. data/sig/ruby_wasm/ext.rbs +25 -2
  39. data/sig/ruby_wasm/feature_set.rbs +12 -0
  40. data/sig/ruby_wasm/packager.rbs +44 -7
  41. metadata +9 -7
  42. data/builders/wasm32-unknown-emscripten/Dockerfile +0 -43
  43. data/builders/wasm32-unknown-emscripten/entrypoint.sh +0 -7
  44. data/builders/wasm32-unknown-wasi/Dockerfile +0 -47
  45. data/builders/wasm32-unknown-wasi/entrypoint.sh +0 -7
@@ -12,7 +12,7 @@ class RubyWasm::Packager::Core
12
12
 
13
13
  extend Forwardable
14
14
 
15
- def_delegators :build_strategy, :cache_key, :artifact
15
+ def_delegators :build_strategy, :cache_key, :artifact, :build_and_link_exts
16
16
 
17
17
  private
18
18
 
@@ -20,7 +20,7 @@ class RubyWasm::Packager::Core
20
20
  @build_strategy ||=
21
21
  begin
22
22
  has_exts = @packager.specs.any? { |spec| spec.extensions.any? }
23
- if @packager.support_dynamic_linking?
23
+ if @packager.features.support_dynamic_linking?
24
24
  DynamicLinking.new(@packager)
25
25
  else
26
26
  StaticLinking.new(@packager)
@@ -37,6 +37,10 @@ class RubyWasm::Packager::Core
37
37
  raise NotImplementedError
38
38
  end
39
39
 
40
+ def build_and_link_exts(executor)
41
+ raise NotImplementedError
42
+ end
43
+
40
44
  # Array of paths to extconf.rb files.
41
45
  def specs_with_extensions
42
46
  @packager.specs.filter_map do |spec|
@@ -61,6 +65,167 @@ class RubyWasm::Packager::Core
61
65
  end
62
66
 
63
67
  class DynamicLinking < BuildStrategy
68
+ def build(executor, options)
69
+ build = derive_build
70
+ force_rebuild =
71
+ options[:remake] || options[:clean] || options[:reconfigure]
72
+ if File.exist?(build.crossruby.artifact) && !force_rebuild
73
+ # Always build extensions because they are usually not expensive to build
74
+ return build.crossruby.artifact
75
+ end
76
+ build.crossruby.clean(executor) if options[:clean]
77
+
78
+ do_build =
79
+ proc do
80
+ build.crossruby.build(
81
+ executor,
82
+ remake: options[:remake],
83
+ reconfigure: options[:reconfigure]
84
+ )
85
+ end
86
+
87
+ __skip__ =
88
+ if defined?(Bundler)
89
+ Bundler.with_unbundled_env(&do_build)
90
+ else
91
+ do_build.call
92
+ end
93
+ build.crossruby.artifact
94
+ end
95
+
96
+ def build_and_link_exts(executor)
97
+ build = derive_build
98
+ self.build_exts(executor, build)
99
+ self.link_exts(executor, build)
100
+ end
101
+
102
+ def link_exts(executor, build)
103
+ ruby_root = build.crossruby.dest_dir
104
+
105
+ libraries = [File.join(ruby_root, "usr", "local", "bin", "ruby")]
106
+
107
+ # TODO: Should be computed from dyinfo of ruby binary
108
+ wasi_libc_shared_libs = [
109
+ "libc.so",
110
+ "libwasi-emulated-getpid.so",
111
+ "libwasi-emulated-mman.so",
112
+ "libwasi-emulated-process-clocks.so",
113
+ "libwasi-emulated-signal.so",
114
+ ]
115
+
116
+ wasi_libc_shared_libs.each do |lib|
117
+ # @type var toolchain: RubyWasm::WASISDK
118
+ toolchain = build.toolchain
119
+ wasi_sdk_path = toolchain.wasi_sdk_path
120
+ libraries << File.join(wasi_sdk_path, "share/wasi-sysroot/lib/wasm32-wasi", lib)
121
+ end
122
+ wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1("command")
123
+ adapters = [wasi_adapter]
124
+ dl_openable_libs = Dir.glob(File.join(ruby_root, "usr", "local", "lib", "ruby", "**", "*.so"))
125
+ linker = RubyWasmExt::ComponentLink.new
126
+ linker.use_built_in_libdl(true)
127
+ linker.stub_missing_functions(false)
128
+ linker.validate(true)
129
+
130
+ libraries.each do |lib|
131
+ # Non-DL openable libraries should be referenced as base name
132
+ lib_name = File.basename(lib)
133
+ module_bytes = File.binread(lib)
134
+ RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
135
+ linker.library(lib_name, module_bytes, false)
136
+ end
137
+
138
+ dl_openable_libs.each do |lib|
139
+ # DL openable lib_name should be a relative path from ruby_root
140
+ lib_name = "/" + Pathname.new(lib).relative_path_from(Pathname.new(ruby_root)).to_s
141
+ module_bytes = File.binread(lib)
142
+ RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
143
+ linker.library(lib_name, module_bytes, true)
144
+ end
145
+
146
+ adapters.each do |adapter|
147
+ adapter_name = File.basename(adapter)
148
+ # e.g. wasi_snapshot_preview1.command.wasm -> wasi_snapshot_preview1
149
+ adapter_name = adapter_name.split(".")[0]
150
+ module_bytes = File.binread(adapter)
151
+ linker.adapter(adapter_name, module_bytes)
152
+ end
153
+ return linker.encode()
154
+ end
155
+
156
+ def build_exts(executor, build)
157
+ exts = specs_with_extensions.flat_map do |spec, exts|
158
+ exts.map do |ext|
159
+ ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape"
160
+ ext_srcdir = File.join(spec.full_gem_path, ext_feature)
161
+ ext_relative_path = File.join(spec.full_name, ext_feature)
162
+ RubyWasm::CrossRubyExtProduct.new(
163
+ ext_srcdir,
164
+ build.toolchain,
165
+ ext_relative_path: ext_relative_path
166
+ )
167
+ end
168
+ end
169
+
170
+ exts.each do |prod|
171
+ executor.begin_section prod.class, prod.name, "Building"
172
+ prod.build(executor, build.crossruby)
173
+ executor.end_section prod.class, prod.name
174
+ end
175
+ end
176
+
177
+ def cache_key(digest)
178
+ derive_build.cache_key(digest)
179
+ end
180
+
181
+ def artifact
182
+ derive_build.crossruby.artifact
183
+ end
184
+
185
+ def target
186
+ RubyWasm::Target.new(@packager.full_build_options[:target], pic: true)
187
+ end
188
+
189
+ def derive_build
190
+ return @build if @build
191
+ __skip__ =
192
+ build ||= RubyWasm::Build.new(
193
+ name, **@packager.full_build_options,
194
+ target: target,
195
+ # NOTE: We don't need linking libwasi_vfs because we use wasi-virt instead.
196
+ wasi_vfs: nil
197
+ )
198
+ build.crossruby.cflags = %w[-fPIC -fvisibility=default]
199
+ if @packager.full_build_options[:target] != "wasm32-unknown-emscripten"
200
+ build.crossruby.debugflags = %w[-g]
201
+ build.crossruby.wasmoptflags = %w[-O3 -g --pass-arg=asyncify-relocatable]
202
+ build.crossruby.ldflags = %w[
203
+ -Xlinker
204
+ --stack-first
205
+ -Xlinker
206
+ -z
207
+ -Xlinker
208
+ stack-size=16777216
209
+ ]
210
+ build.crossruby.xldflags = %w[
211
+ -Xlinker -shared
212
+ -Xlinker --export-dynamic
213
+ -Xlinker --export-all
214
+ -Xlinker --experimental-pic
215
+ -Xlinker -export-if-defined=__main_argc_argv
216
+ ]
217
+ end
218
+ @build = build
219
+ build
220
+ end
221
+
222
+ def name
223
+ require "digest"
224
+ options = @packager.full_build_options
225
+ src_channel = options[:src][:name]
226
+ target_triplet = options[:target]
227
+ "ruby-#{src_channel}-#{target_triplet}-pic#{options[:suffix]}"
228
+ end
64
229
  end
65
230
 
66
231
  class StaticLinking < BuildStrategy
@@ -99,10 +264,14 @@ class RubyWasm::Packager::Core
99
264
  derive_build.crossruby.artifact
100
265
  end
101
266
 
267
+ def target
268
+ RubyWasm::Target.new(@packager.full_build_options[:target])
269
+ end
270
+
102
271
  def derive_build
103
272
  return @build if @build
104
273
  __skip__ =
105
- build ||= RubyWasm::Build.new(name, **@packager.full_build_options)
274
+ build ||= RubyWasm::Build.new(name, **@packager.full_build_options, target: target)
106
275
  build.crossruby.user_exts = user_exts(build)
107
276
  # Emscripten uses --global-base=1024 by default, but it conflicts with
108
277
  # --stack-first and -z stack-size since global-base 1024 is smaller than
@@ -111,7 +280,9 @@ class RubyWasm::Packager::Core
111
280
  # script of Ruby.
112
281
  if @packager.full_build_options[:target] != "wasm32-unknown-emscripten"
113
282
  build.crossruby.debugflags = %w[-g]
114
- build.crossruby.wasmoptflags = %w[-O3 -g]
283
+ # We assume that imported functions provided through WASI will not change
284
+ # asyncify state, so we ignore them.
285
+ build.crossruby.wasmoptflags = %w[-O3 -g --pass-arg=asyncify-ignore-imports]
115
286
  build.crossruby.ldflags = %w[
116
287
  -Xlinker
117
288
  --stack-first
@@ -125,6 +296,23 @@ class RubyWasm::Packager::Core
125
296
  build
126
297
  end
127
298
 
299
+ def build_and_link_exts(executor)
300
+ build = derive_build
301
+ ruby_root = build.crossruby.dest_dir
302
+ module_bytes = File.binread(File.join(ruby_root, "usr", "local", "bin", "ruby"))
303
+ return module_bytes unless @packager.features.support_component_model?
304
+
305
+ linker = RubyWasmExt::ComponentEncode.new
306
+ linker.validate(true)
307
+ linker.module(module_bytes)
308
+ linker.adapter(
309
+ "wasi_snapshot_preview1",
310
+ File.binread(RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1("reactor"))
311
+ )
312
+
313
+ linker.encode()
314
+ end
315
+
128
316
  def user_exts(build)
129
317
  @user_exts ||=
130
318
  specs_with_extensions.flat_map do |spec, exts|
@@ -65,12 +65,14 @@ class RubyWasm::Packager::FileSystem
65
65
  end
66
66
 
67
67
  def remove_non_runtime_files(executor)
68
- %w[
69
- **/*.so
68
+ patterns = %w[
70
69
  usr/local/lib/libruby-static.a
71
70
  usr/local/bin/ruby
72
71
  usr/local/include
73
- ].each do |pattern|
72
+ ]
73
+
74
+ patterns << "**/*.so" unless @packager.features.support_dynamic_linking?
75
+ patterns.each do |pattern|
74
76
  Dir
75
77
  .glob(File.join(@dest_dir, pattern))
76
78
  .each do |entry|
@@ -2,11 +2,19 @@
2
2
  class RubyWasm::Packager
3
3
  # Initializes a new instance of the RubyWasm::Packager class.
4
4
  #
5
+ # @param root [String] The root directory of the Ruby project.
6
+ # The root directory (will) contain the following files:
7
+ # * build_manifest.json
8
+ # * rubies
9
+ # * build
5
10
  # @param config [Hash] The build config used for building Ruby.
6
11
  # @param definition [Bundler::Definition] The Bundler definition.
7
- def initialize(config = nil, definition = nil)
12
+ # @param features [RubyWasm::FeatureSet] The features used for packaging.
13
+ def initialize(root, config = nil, definition = nil, features: RubyWasm::FeatureSet.derive_from_env)
14
+ @root = root
8
15
  @definition = definition
9
16
  @config = config
17
+ @features = features
10
18
  end
11
19
 
12
20
  # Packages the Ruby code into a Wasm binary. (including extensions)
@@ -21,15 +29,13 @@ class RubyWasm::Packager
21
29
 
22
30
  fs = RubyWasm::Packager::FileSystem.new(dest_dir, self)
23
31
  fs.package_ruby_root(tarball, executor)
24
-
25
- ruby_wasm_bin = File.expand_path("bin/ruby", fs.ruby_root)
26
- wasm_bytes = File.binread(ruby_wasm_bin).bytes
32
+ wasm_bytes = ruby_core.build_and_link_exts(executor)
27
33
 
28
34
  fs.package_gems
29
35
  fs.remove_non_runtime_files(executor)
30
36
  fs.remove_stdlib(executor) unless options[:stdlib]
31
37
 
32
- if full_build_options[:target] == "wasm32-unknown-wasi"
38
+ if full_build_options[:target] == "wasm32-unknown-wasip1" && !features.support_component_model?
33
39
  # wasi-vfs supports only WASI target
34
40
  wasi_vfs = RubyWasmExt::WasiVfs.new
35
41
  wasi_vfs.map_dir("/bundle", fs.bundle_dir)
@@ -52,79 +58,22 @@ class RubyWasm::Packager
52
58
  # Retrieves the specs from the Bundler definition, excluding the excluded gems.
53
59
  def specs
54
60
  return [] unless @definition
55
- @definition.specs.reject { |spec| EXCLUDED_GEMS.include?(spec.name) }
56
- end
57
-
58
- # Checks if dynamic linking is supported.
59
- def support_dynamic_linking?
60
- @ruby_channel == "head"
61
- end
62
-
63
- # Retrieves the root directory of the Ruby project.
64
- # The root directory contains the following stuff:
65
- # * patches/*.patch
66
- # * build_manifest.json
67
- # * rubies
68
- # * build
69
- def root
70
- __skip__ =
71
- @root ||=
72
- begin
73
- if explicit = ENV["RUBY_WASM_ROOT"]
74
- File.expand_path(explicit)
75
- elsif defined?(Bundler)
76
- Bundler.root
77
- else
78
- Dir.pwd
79
- end
80
- rescue Bundler::GemfileNotFound
81
- Dir.pwd
82
- end
61
+ @specs ||= @definition.resolve.materialize(@definition.requested_dependencies)
62
+ .reject { |spec| EXCLUDED_GEMS.include?(spec.name) }
63
+ @specs
83
64
  end
84
65
 
85
- # Retrieves the alias definitions for the Ruby sources.
86
- def self.build_source_aliases(root)
87
- patches = Dir[File.join(root, "patches", "*.patch")]
88
- sources = {
89
- "head" => {
90
- type: "github",
91
- repo: "ruby/ruby",
92
- rev: "master",
93
- patches: patches.map { |p| File.expand_path(p) }
94
- },
95
- "3.3" => {
96
- type: "tarball",
97
- url: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0.tar.gz"
98
- },
99
- "3.2" => {
100
- type: "tarball",
101
- url: "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.3.tar.gz"
102
- }
103
- }
104
- sources.each { |name, source| source[:name] = name }
105
-
106
- build_manifest = File.join(root, "build_manifest.json")
107
- if File.exist?(build_manifest)
108
- begin
109
- manifest = JSON.parse(File.read(build_manifest))
110
- manifest["ruby_revisions"].each do |name, rev|
111
- sources[name][:rev] = rev
112
- end
113
- rescue StandardError => e
114
- RubyWasm.logger.warn "Failed to load build_manifest.json: #{e}"
115
- end
116
- end
117
- sources
66
+ def features
67
+ @features
118
68
  end
119
69
 
120
70
  ALL_DEFAULT_EXTS =
121
- "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl"
71
+ "cgi/escape,continuation,coverage,date,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,json,json/generator,json/parser,objspace,pathname,psych,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl"
122
72
 
123
73
  # Retrieves the build options used for building Ruby itself.
124
74
  def build_options
125
75
  default = {
126
- target: "wasm32-unknown-wasi",
127
- src: "3.3",
76
+ target: RubyWasm::Target.new("wasm32-unknown-wasip1"),
128
77
  default_exts: ALL_DEFAULT_EXTS
129
78
  }
130
79
  override = @config || {}
@@ -135,25 +84,14 @@ class RubyWasm::Packager
135
84
  # Retrieves the resolved build options
136
85
  def full_build_options
137
86
  options = build_options
138
- build_dir = File.join(root, "build")
139
- rubies_dir = File.join(root, "rubies")
87
+ build_dir = File.join(@root, "build")
88
+ rubies_dir = File.join(@root, "rubies")
140
89
  toolchain = RubyWasm::Toolchain.get(options[:target], build_dir)
141
- src =
142
- if options[:src].is_a?(Hash)
143
- options[:src]
144
- else
145
- src_name = options[:src]
146
- aliases = self.class.build_source_aliases(root)
147
- aliases[src_name] ||
148
- raise(
149
- "Unknown Ruby source: #{src_name} (available: #{aliases.keys.join(", ")})"
150
- )
151
- end
152
90
  options.merge(
153
91
  toolchain: toolchain,
154
92
  build_dir: build_dir,
155
93
  rubies_dir: rubies_dir,
156
- src: src
94
+ src: options[:src]
157
95
  )
158
96
  end
159
97
  end
@@ -16,6 +16,7 @@ class RubyWasm::BuildTask < ::Rake::TaskLib
16
16
  **options,
17
17
  &block
18
18
  )
19
+ target = Target.new(target)
19
20
  @build =
20
21
  RubyWasm::Build.new(
21
22
  name,
@@ -1,3 +1,3 @@
1
1
  module RubyWasm
2
- VERSION = "2.5.0"
2
+ VERSION = "2.5.1"
3
3
  end
data/lib/ruby_wasm.rb CHANGED
@@ -3,7 +3,9 @@ require "logger"
3
3
  require_relative "ruby_wasm/version"
4
4
  require_relative "ruby_wasm/util"
5
5
  require_relative "ruby_wasm/build"
6
+ require_relative "ruby_wasm/feature_set"
6
7
  require_relative "ruby_wasm/packager"
8
+ require_relative "ruby_wasm/packager/component_adapter"
7
9
  require_relative "ruby_wasm/packager/file_system"
8
10
  require_relative "ruby_wasm/packager/core"
9
11