ruby_wasm 2.5.0 → 2.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/CONTRIBUTING.md +7 -7
  3. data/Cargo.lock +91 -6
  4. data/Gemfile +1 -1
  5. data/README.md +9 -9
  6. data/Rakefile +9 -7
  7. data/docs/cheat_sheet.md +8 -8
  8. data/ext/ruby_wasm/Cargo.toml +3 -1
  9. data/ext/ruby_wasm/src/lib.rs +198 -8
  10. data/lib/ruby_wasm/build/executor.rb +4 -0
  11. data/lib/ruby_wasm/build/product/crossruby.rb +59 -25
  12. data/lib/ruby_wasm/build/product/libyaml.rb +5 -3
  13. data/lib/ruby_wasm/build/product/openssl.rb +7 -2
  14. data/lib/ruby_wasm/build/product/product.rb +3 -3
  15. data/lib/ruby_wasm/build/product/ruby_source.rb +3 -3
  16. data/lib/ruby_wasm/build/product/wasi_vfs.rb +1 -1
  17. data/lib/ruby_wasm/build/product/zlib.rb +3 -1
  18. data/lib/ruby_wasm/build/target.rb +24 -0
  19. data/lib/ruby_wasm/build/toolchain/wit_bindgen.rb +2 -2
  20. data/lib/ruby_wasm/build/toolchain.rb +1 -1
  21. data/lib/ruby_wasm/build.rb +7 -3
  22. data/lib/ruby_wasm/cli.rb +147 -11
  23. data/lib/ruby_wasm/feature_set.rb +30 -0
  24. data/lib/ruby_wasm/packager/component_adapter/wasi_snapshot_preview1.command.wasm +0 -0
  25. data/lib/ruby_wasm/packager/component_adapter/wasi_snapshot_preview1.reactor.wasm +0 -0
  26. data/lib/ruby_wasm/packager/component_adapter.rb +14 -0
  27. data/lib/ruby_wasm/packager/core.rb +199 -5
  28. data/lib/ruby_wasm/packager/file_system.rb +5 -3
  29. data/lib/ruby_wasm/packager.rb +22 -82
  30. data/lib/ruby_wasm/rake_task.rb +1 -0
  31. data/lib/ruby_wasm/version.rb +1 -1
  32. data/lib/ruby_wasm.rb +2 -0
  33. data/package-lock.json +5571 -7015
  34. data/package.json +3 -3
  35. data/rakelib/check.rake +23 -10
  36. data/rakelib/ci.rake +3 -3
  37. data/rakelib/packaging.rake +44 -15
  38. data/sig/ruby_wasm/build.rbs +38 -28
  39. data/sig/ruby_wasm/cli.rbs +27 -3
  40. data/sig/ruby_wasm/ext.rbs +25 -2
  41. data/sig/ruby_wasm/feature_set.rbs +12 -0
  42. data/sig/ruby_wasm/packager.rbs +44 -7
  43. metadata +9 -10
  44. data/builders/wasm32-unknown-emscripten/Dockerfile +0 -43
  45. data/builders/wasm32-unknown-emscripten/entrypoint.sh +0 -7
  46. data/builders/wasm32-unknown-wasi/Dockerfile +0 -47
  47. data/builders/wasm32-unknown-wasi/entrypoint.sh +0 -7
@@ -0,0 +1,14 @@
1
+ module RubyWasm::Packager::ComponentAdapter
2
+ module_function
3
+
4
+ # The path to the component adapter for the given WASI execution model.
5
+ #
6
+ # @param exec_model [String] "command" or "reactor"
7
+ def wasi_snapshot_preview1(exec_model)
8
+ File.join(
9
+ File.dirname(__FILE__),
10
+ "component_adapter",
11
+ "wasi_snapshot_preview1.#{exec_model}.wasm"
12
+ )
13
+ end
14
+ end
@@ -12,7 +12,7 @@ class RubyWasm::Packager::Core
12
12
 
13
13
  extend Forwardable
14
14
 
15
- def_delegators :build_strategy, :cache_key, :artifact
15
+ def_delegators :build_strategy, :cache_key, :artifact, :build_and_link_exts
16
16
 
17
17
  private
18
18
 
@@ -20,7 +20,7 @@ class RubyWasm::Packager::Core
20
20
  @build_strategy ||=
21
21
  begin
22
22
  has_exts = @packager.specs.any? { |spec| spec.extensions.any? }
23
- if @packager.support_dynamic_linking?
23
+ if @packager.features.support_dynamic_linking?
24
24
  DynamicLinking.new(@packager)
25
25
  else
26
26
  StaticLinking.new(@packager)
@@ -37,6 +37,10 @@ class RubyWasm::Packager::Core
37
37
  raise NotImplementedError
38
38
  end
39
39
 
40
+ def build_and_link_exts(executor, module_bytes)
41
+ raise NotImplementedError
42
+ end
43
+
40
44
  # Array of paths to extconf.rb files.
41
45
  def specs_with_extensions
42
46
  @packager.specs.filter_map do |spec|
@@ -61,6 +65,168 @@ class RubyWasm::Packager::Core
61
65
  end
62
66
 
63
67
  class DynamicLinking < BuildStrategy
68
+ def build(executor, options)
69
+ build = derive_build
70
+ force_rebuild =
71
+ options[:remake] || options[:clean] || options[:reconfigure]
72
+ if File.exist?(build.crossruby.artifact) && !force_rebuild
73
+ # Always build extensions because they are usually not expensive to build
74
+ return build.crossruby.artifact
75
+ end
76
+ build.crossruby.clean(executor) if options[:clean]
77
+
78
+ do_build =
79
+ proc do
80
+ build.crossruby.build(
81
+ executor,
82
+ remake: options[:remake],
83
+ reconfigure: options[:reconfigure]
84
+ )
85
+ end
86
+
87
+ __skip__ =
88
+ if defined?(Bundler)
89
+ Bundler.with_unbundled_env(&do_build)
90
+ else
91
+ do_build.call
92
+ end
93
+ build.crossruby.artifact
94
+ end
95
+
96
+ def build_and_link_exts(executor, module_bytes)
97
+ build = derive_build
98
+ self.build_exts(executor, build)
99
+ self.link_exts(executor, build)
100
+ end
101
+
102
+ def link_exts(executor, build)
103
+ ruby_root = build.crossruby.dest_dir
104
+
105
+ libraries = [File.join(ruby_root, "usr", "local", "bin", "ruby")]
106
+
107
+ # TODO: Should be computed from dyinfo of ruby binary
108
+ wasi_libc_shared_libs = [
109
+ "libc.so",
110
+ "libwasi-emulated-getpid.so",
111
+ "libwasi-emulated-mman.so",
112
+ "libwasi-emulated-process-clocks.so",
113
+ "libwasi-emulated-signal.so",
114
+ ]
115
+
116
+ wasi_libc_shared_libs.each do |lib|
117
+ # @type var toolchain: RubyWasm::WASISDK
118
+ toolchain = build.toolchain
119
+ wasi_sdk_path = toolchain.wasi_sdk_path
120
+ libraries << File.join(wasi_sdk_path, "share/wasi-sysroot/lib/wasm32-wasi", lib)
121
+ end
122
+ wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1("command")
123
+ adapters = [wasi_adapter]
124
+ dl_openable_libs = Dir.glob(File.join(ruby_root, "usr", "local", "lib", "ruby", "**", "*.so"))
125
+ linker = RubyWasmExt::ComponentLink.new
126
+ linker.use_built_in_libdl(true)
127
+ linker.stub_missing_functions(false)
128
+ linker.validate(true)
129
+
130
+ libraries.each do |lib|
131
+ # Non-DL openable libraries should be referenced as base name
132
+ lib_name = File.basename(lib)
133
+ module_bytes = File.binread(lib)
134
+ RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
135
+ linker.library(lib_name, module_bytes, false)
136
+ end
137
+
138
+ dl_openable_libs.each do |lib|
139
+ # DL openable lib_name should be a relative path from ruby_root
140
+ lib_name = "/" + Pathname.new(lib).relative_path_from(Pathname.new(ruby_root)).to_s
141
+ module_bytes = File.binread(lib)
142
+ RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
143
+ linker.library(lib_name, module_bytes, true)
144
+ end
145
+
146
+ adapters.each do |adapter|
147
+ adapter_name = File.basename(adapter)
148
+ # e.g. wasi_snapshot_preview1.command.wasm -> wasi_snapshot_preview1
149
+ adapter_name = adapter_name.split(".")[0]
150
+ module_bytes = File.binread(adapter)
151
+ linker.adapter(adapter_name, module_bytes)
152
+ end
153
+ return linker.encode()
154
+ end
155
+
156
+ def build_exts(executor, build)
157
+ exts = specs_with_extensions.flat_map do |spec, exts|
158
+ exts.map do |ext|
159
+ ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape"
160
+ ext_srcdir = File.join(spec.full_gem_path, ext_feature)
161
+ ext_relative_path = File.join(spec.full_name, ext_feature)
162
+ RubyWasm::CrossRubyExtProduct.new(
163
+ ext_srcdir,
164
+ build.toolchain,
165
+ features: @packager.features,
166
+ ext_relative_path: ext_relative_path
167
+ )
168
+ end
169
+ end
170
+
171
+ exts.each do |prod|
172
+ executor.begin_section prod.class, prod.name, "Building"
173
+ prod.build(executor, build.crossruby)
174
+ executor.end_section prod.class, prod.name
175
+ end
176
+ end
177
+
178
+ def cache_key(digest)
179
+ derive_build.cache_key(digest)
180
+ end
181
+
182
+ def artifact
183
+ derive_build.crossruby.artifact
184
+ end
185
+
186
+ def target
187
+ RubyWasm::Target.new(@packager.full_build_options[:target], pic: true)
188
+ end
189
+
190
+ def derive_build
191
+ return @build if @build
192
+ __skip__ =
193
+ build ||= RubyWasm::Build.new(
194
+ name, **@packager.full_build_options,
195
+ target: target,
196
+ # NOTE: We don't need linking libwasi_vfs because we use wasi-virt instead.
197
+ wasi_vfs: nil
198
+ )
199
+ build.crossruby.cflags = %w[-fPIC -fvisibility=default]
200
+ if @packager.full_build_options[:target] != "wasm32-unknown-emscripten"
201
+ build.crossruby.debugflags = %w[-g]
202
+ build.crossruby.wasmoptflags = %w[-O3 -g --pass-arg=asyncify-relocatable]
203
+ build.crossruby.ldflags = %w[
204
+ -Xlinker
205
+ --stack-first
206
+ -Xlinker
207
+ -z
208
+ -Xlinker
209
+ stack-size=16777216
210
+ ]
211
+ build.crossruby.xldflags = %w[
212
+ -Xlinker -shared
213
+ -Xlinker --export-dynamic
214
+ -Xlinker --export-all
215
+ -Xlinker --experimental-pic
216
+ -Xlinker -export-if-defined=__main_argc_argv
217
+ ]
218
+ end
219
+ @build = build
220
+ build
221
+ end
222
+
223
+ def name
224
+ require "digest"
225
+ options = @packager.full_build_options
226
+ src_channel = options[:src][:name]
227
+ target_triplet = options[:target]
228
+ "ruby-#{src_channel}-#{target_triplet}-pic#{options[:suffix]}"
229
+ end
64
230
  end
65
231
 
66
232
  class StaticLinking < BuildStrategy
@@ -93,16 +259,24 @@ class RubyWasm::Packager::Core
93
259
 
94
260
  def cache_key(digest)
95
261
  derive_build.cache_key(digest)
262
+ if enabled = @packager.features.support_component_model?
263
+ digest << enabled.to_s
264
+ end
96
265
  end
97
266
 
98
267
  def artifact
99
268
  derive_build.crossruby.artifact
100
269
  end
101
270
 
271
+ def target
272
+ RubyWasm::Target.new(@packager.full_build_options[:target])
273
+ end
274
+
102
275
  def derive_build
103
276
  return @build if @build
104
- __skip__ =
105
- build ||= RubyWasm::Build.new(name, **@packager.full_build_options)
277
+ __skip__ = build ||= RubyWasm::Build.new(
278
+ name, **@packager.full_build_options, target: target,
279
+ )
106
280
  build.crossruby.user_exts = user_exts(build)
107
281
  # Emscripten uses --global-base=1024 by default, but it conflicts with
108
282
  # --stack-first and -z stack-size since global-base 1024 is smaller than
@@ -111,7 +285,9 @@ class RubyWasm::Packager::Core
111
285
  # script of Ruby.
112
286
  if @packager.full_build_options[:target] != "wasm32-unknown-emscripten"
113
287
  build.crossruby.debugflags = %w[-g]
114
- build.crossruby.wasmoptflags = %w[-O3 -g]
288
+ # We assume that imported functions provided through WASI will not change
289
+ # asyncify state, so we ignore them.
290
+ build.crossruby.wasmoptflags = %w[-O3 -g --pass-arg=asyncify-ignore-imports]
115
291
  build.crossruby.ldflags = %w[
116
292
  -Xlinker
117
293
  --stack-first
@@ -125,6 +301,20 @@ class RubyWasm::Packager::Core
125
301
  build
126
302
  end
127
303
 
304
+ def build_and_link_exts(executor, module_bytes)
305
+ return module_bytes unless @packager.features.support_component_model?
306
+
307
+ linker = RubyWasmExt::ComponentEncode.new
308
+ linker.validate(true)
309
+ linker.module(module_bytes)
310
+ linker.adapter(
311
+ "wasi_snapshot_preview1",
312
+ File.binread(RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1("reactor"))
313
+ )
314
+
315
+ linker.encode()
316
+ end
317
+
128
318
  def user_exts(build)
129
319
  @user_exts ||=
130
320
  specs_with_extensions.flat_map do |spec, exts|
@@ -135,6 +325,7 @@ class RubyWasm::Packager::Core
135
325
  RubyWasm::CrossRubyExtProduct.new(
136
326
  ext_srcdir,
137
327
  build.toolchain,
328
+ features: @packager.features,
138
329
  ext_relative_path: ext_relative_path
139
330
  )
140
331
  end
@@ -150,6 +341,9 @@ class RubyWasm::Packager::Core
150
341
  exts = specs_with_extensions.sort
151
342
  hash = ::Digest::MD5.new
152
343
  specs_with_extensions.each { |spec, _| hash << spec.full_name }
344
+ if enabled = @packager.features.support_component_model?
345
+ hash << enabled.to_s
346
+ end
153
347
  exts.empty? ? base : "#{base}-#{hash.hexdigest}"
154
348
  end
155
349
  end
@@ -65,12 +65,14 @@ class RubyWasm::Packager::FileSystem
65
65
  end
66
66
 
67
67
  def remove_non_runtime_files(executor)
68
- %w[
69
- **/*.so
68
+ patterns = %w[
70
69
  usr/local/lib/libruby-static.a
71
70
  usr/local/bin/ruby
72
71
  usr/local/include
73
- ].each do |pattern|
72
+ ]
73
+
74
+ patterns << "**/*.so" unless @packager.features.support_dynamic_linking?
75
+ patterns.each do |pattern|
74
76
  Dir
75
77
  .glob(File.join(@dest_dir, pattern))
76
78
  .each do |entry|
@@ -2,11 +2,19 @@
2
2
  class RubyWasm::Packager
3
3
  # Initializes a new instance of the RubyWasm::Packager class.
4
4
  #
5
+ # @param root [String] The root directory of the Ruby project.
6
+ # The root directory (will) contain the following files:
7
+ # * build_manifest.json
8
+ # * rubies
9
+ # * build
5
10
  # @param config [Hash] The build config used for building Ruby.
6
11
  # @param definition [Bundler::Definition] The Bundler definition.
7
- def initialize(config = nil, definition = nil)
12
+ # @param features [RubyWasm::FeatureSet] The features used for packaging.
13
+ def initialize(root, config = nil, definition = nil, features: RubyWasm::FeatureSet.derive_from_env)
14
+ @root = root
8
15
  @definition = definition
9
16
  @config = config
17
+ @features = features
10
18
  end
11
19
 
12
20
  # Packages the Ruby code into a Wasm binary. (including extensions)
@@ -22,14 +30,13 @@ class RubyWasm::Packager
22
30
  fs = RubyWasm::Packager::FileSystem.new(dest_dir, self)
23
31
  fs.package_ruby_root(tarball, executor)
24
32
 
25
- ruby_wasm_bin = File.expand_path("bin/ruby", fs.ruby_root)
26
- wasm_bytes = File.binread(ruby_wasm_bin).bytes
33
+ wasm_bytes = File.binread(File.join(fs.ruby_root, "bin", "ruby"))
27
34
 
28
35
  fs.package_gems
29
36
  fs.remove_non_runtime_files(executor)
30
37
  fs.remove_stdlib(executor) unless options[:stdlib]
31
38
 
32
- if full_build_options[:target] == "wasm32-unknown-wasi"
39
+ if full_build_options[:target] == "wasm32-unknown-wasip1"
33
40
  # wasi-vfs supports only WASI target
34
41
  wasi_vfs = RubyWasmExt::WasiVfs.new
35
42
  wasi_vfs.map_dir("/bundle", fs.bundle_dir)
@@ -37,6 +44,7 @@ class RubyWasm::Packager
37
44
 
38
45
  wasm_bytes = wasi_vfs.pack(wasm_bytes)
39
46
  end
47
+ wasm_bytes = ruby_core.build_and_link_exts(executor, wasm_bytes)
40
48
 
41
49
  wasm_bytes = RubyWasmExt.preinitialize(wasm_bytes) if options[:optimize]
42
50
  wasm_bytes
@@ -52,79 +60,22 @@ class RubyWasm::Packager
52
60
  # Retrieves the specs from the Bundler definition, excluding the excluded gems.
53
61
  def specs
54
62
  return [] unless @definition
55
- @definition.specs.reject { |spec| EXCLUDED_GEMS.include?(spec.name) }
63
+ @specs ||= @definition.resolve.materialize(@definition.requested_dependencies)
64
+ .reject { |spec| EXCLUDED_GEMS.include?(spec.name) }
65
+ @specs
56
66
  end
57
67
 
58
- # Checks if dynamic linking is supported.
59
- def support_dynamic_linking?
60
- @ruby_channel == "head"
61
- end
62
-
63
- # Retrieves the root directory of the Ruby project.
64
- # The root directory contains the following stuff:
65
- # * patches/*.patch
66
- # * build_manifest.json
67
- # * rubies
68
- # * build
69
- def root
70
- __skip__ =
71
- @root ||=
72
- begin
73
- if explicit = ENV["RUBY_WASM_ROOT"]
74
- File.expand_path(explicit)
75
- elsif defined?(Bundler)
76
- Bundler.root
77
- else
78
- Dir.pwd
79
- end
80
- rescue Bundler::GemfileNotFound
81
- Dir.pwd
82
- end
83
- end
84
-
85
- # Retrieves the alias definitions for the Ruby sources.
86
- def self.build_source_aliases(root)
87
- patches = Dir[File.join(root, "patches", "*.patch")]
88
- sources = {
89
- "head" => {
90
- type: "github",
91
- repo: "ruby/ruby",
92
- rev: "master",
93
- patches: patches.map { |p| File.expand_path(p) }
94
- },
95
- "3.3" => {
96
- type: "tarball",
97
- url: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0.tar.gz"
98
- },
99
- "3.2" => {
100
- type: "tarball",
101
- url: "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.3.tar.gz"
102
- }
103
- }
104
- sources.each { |name, source| source[:name] = name }
105
-
106
- build_manifest = File.join(root, "build_manifest.json")
107
- if File.exist?(build_manifest)
108
- begin
109
- manifest = JSON.parse(File.read(build_manifest))
110
- manifest["ruby_revisions"].each do |name, rev|
111
- sources[name][:rev] = rev
112
- end
113
- rescue StandardError => e
114
- RubyWasm.logger.warn "Failed to load build_manifest.json: #{e}"
115
- end
116
- end
117
- sources
68
+ def features
69
+ @features
118
70
  end
119
71
 
120
72
  ALL_DEFAULT_EXTS =
121
- "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl"
73
+ "cgi/escape,continuation,coverage,date,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,json,json/generator,json/parser,objspace,pathname,psych,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl"
122
74
 
123
75
  # Retrieves the build options used for building Ruby itself.
124
76
  def build_options
125
77
  default = {
126
- target: "wasm32-unknown-wasi",
127
- src: "3.3",
78
+ target: RubyWasm::Target.new("wasm32-unknown-wasip1"),
128
79
  default_exts: ALL_DEFAULT_EXTS
129
80
  }
130
81
  override = @config || {}
@@ -135,25 +86,14 @@ class RubyWasm::Packager
135
86
  # Retrieves the resolved build options
136
87
  def full_build_options
137
88
  options = build_options
138
- build_dir = File.join(root, "build")
139
- rubies_dir = File.join(root, "rubies")
89
+ build_dir = File.join(@root, "build")
90
+ rubies_dir = File.join(@root, "rubies")
140
91
  toolchain = RubyWasm::Toolchain.get(options[:target], build_dir)
141
- src =
142
- if options[:src].is_a?(Hash)
143
- options[:src]
144
- else
145
- src_name = options[:src]
146
- aliases = self.class.build_source_aliases(root)
147
- aliases[src_name] ||
148
- raise(
149
- "Unknown Ruby source: #{src_name} (available: #{aliases.keys.join(", ")})"
150
- )
151
- end
152
92
  options.merge(
153
93
  toolchain: toolchain,
154
94
  build_dir: build_dir,
155
95
  rubies_dir: rubies_dir,
156
- src: src
96
+ src: options[:src]
157
97
  )
158
98
  end
159
99
  end
@@ -16,6 +16,7 @@ class RubyWasm::BuildTask < ::Rake::TaskLib
16
16
  **options,
17
17
  &block
18
18
  )
19
+ target = Target.new(target)
19
20
  @build =
20
21
  RubyWasm::Build.new(
21
22
  name,
@@ -1,3 +1,3 @@
1
1
  module RubyWasm
2
- VERSION = "2.5.0"
2
+ VERSION = "2.5.2"
3
3
  end
data/lib/ruby_wasm.rb CHANGED
@@ -3,7 +3,9 @@ require "logger"
3
3
  require_relative "ruby_wasm/version"
4
4
  require_relative "ruby_wasm/util"
5
5
  require_relative "ruby_wasm/build"
6
+ require_relative "ruby_wasm/feature_set"
6
7
  require_relative "ruby_wasm/packager"
8
+ require_relative "ruby_wasm/packager/component_adapter"
7
9
  require_relative "ruby_wasm/packager/file_system"
8
10
  require_relative "ruby_wasm/packager/core"
9
11